diff --git a/pkg/api/apitesting/roundtrip/unstructured.go b/pkg/api/apitesting/roundtrip/unstructured.go index 2c2e97bc8..35be03ac9 100644 --- a/pkg/api/apitesting/roundtrip/unstructured.go +++ b/pkg/api/apitesting/roundtrip/unstructured.go @@ -132,6 +132,23 @@ func RoundtripToUnstructured(t *testing.T, scheme *runtime.Scheme, funcs fuzzer. t.Fatalf("unstructured via json differed from unstructured via cbor: %v", cmp.Diff(uJSON, uCBOR)) } + // original->CBOR(nondeterministic)->Unstructured + buf.Reset() + if err := cborSerializer.EncodeNondeterministic(item, &buf); err != nil { + t.Fatalf("error encoding native to cbor: %v", err) + } + var uCBORNondeterministic runtime.Object = &unstructured.Unstructured{} + uCBORNondeterministic, _, err = cborSerializer.Decode(buf.Bytes(), &gvk, uCBORNondeterministic) + if err != nil { + diag, _ := cbor.Diagnose(buf.Bytes()) + t.Fatalf("error decoding cbor to unstructured: %v, diag: %s", err, diag) + } + + // original->CBOR->Unstructured == original->CBOR(nondeterministic)->Unstructured + if !apiequality.Semantic.DeepEqual(uCBOR, uCBORNondeterministic) { + t.Fatalf("unstructured via nondeterministic cbor differed from unstructured via cbor: %v", cmp.Diff(uCBOR, uCBORNondeterministic)) + } + // original->JSON/CBOR->Unstructured == original->JSON/CBOR->Unstructured->JSON->Unstructured buf.Reset() if err := jsonSerializer.Encode(uJSON, &buf); err != nil { @@ -161,6 +178,21 @@ func RoundtripToUnstructured(t *testing.T, scheme *runtime.Scheme, funcs fuzzer. t.Errorf("object changed during native-cbor-unstructured-cbor-unstructured roundtrip, diff: %s", cmp.Diff(uCBOR, uCBOR2)) } + // original->JSON/CBOR->Unstructured->CBOR->Unstructured == original->JSON/CBOR->Unstructured->CBOR(nondeterministic)->Unstructured + buf.Reset() + if err := cborSerializer.EncodeNondeterministic(uCBOR, &buf); err != nil { + t.Fatalf("error encoding unstructured to cbor: %v", err) + } + var uCBOR2Nondeterministic runtime.Object = &unstructured.Unstructured{} + uCBOR2Nondeterministic, _, err = cborSerializer.Decode(buf.Bytes(), &gvk, uCBOR2Nondeterministic) + if err != nil { + diag, _ := cbor.Diagnose(buf.Bytes()) + t.Fatalf("error decoding cbor to unstructured: %v, diag: %s", err, diag) + } + if !apiequality.Semantic.DeepEqual(uCBOR, uCBOR2Nondeterministic) { + t.Errorf("object changed during native-cbor-unstructured-cbor(nondeterministic)-unstructured roundtrip, diff: %s", cmp.Diff(uCBOR, uCBOR2Nondeterministic)) + } + // original->JSON/CBOR->Unstructured->JSON->final == original buf.Reset() if err := jsonSerializer.Encode(uJSON, &buf); err != nil { @@ -187,6 +219,20 @@ func RoundtripToUnstructured(t *testing.T, scheme *runtime.Scheme, funcs fuzzer. if !apiequality.Semantic.DeepEqual(item, finalCBOR) { t.Errorf("object changed during native-cbor-unstructured-cbor-native roundtrip, diff: %s", cmp.Diff(item, finalCBOR)) } + + // original->JSON/CBOR->Unstructured->CBOR(nondeterministic)->final == original + buf.Reset() + if err := cborSerializer.EncodeNondeterministic(uCBOR, &buf); err != nil { + t.Fatalf("error encoding unstructured to cbor: %v", err) + } + finalCBORNondeterministic, _, err := cborSerializer.Decode(buf.Bytes(), &gvk, nil) + if err != nil { + diag, _ := cbor.Diagnose(buf.Bytes()) + t.Fatalf("error decoding cbor to native: %v, diag: %s", err, diag) + } + if !apiequality.Semantic.DeepEqual(item, finalCBORNondeterministic) { + t.Errorf("object changed during native-cbor-unstructured-cbor-native roundtrip, diff: %s", cmp.Diff(item, finalCBORNondeterministic)) + } } }) } diff --git a/pkg/runtime/interfaces.go b/pkg/runtime/interfaces.go index e89ea8939..2703300cd 100644 --- a/pkg/runtime/interfaces.go +++ b/pkg/runtime/interfaces.go @@ -69,6 +69,19 @@ type Encoder interface { Identifier() Identifier } +// NondeterministicEncoder is implemented by Encoders that can serialize objects more efficiently in +// cases where the output does not need to be deterministic. +type NondeterministicEncoder interface { + Encoder + + // EncodeNondeterministic writes an object to the stream. Unlike the Encode method of + // Encoder, EncodeNondeterministic does not guarantee that any two invocations will write + // the same sequence of bytes to the io.Writer. Any differences will not be significant to a + // generic decoder. For example, map entries and struct fields might be encoded in any + // order. + EncodeNondeterministic(Object, io.Writer) error +} + // MemoryAllocator is responsible for allocating memory. // By encapsulating memory allocation into its own interface, we can reuse the memory // across many operations in places we know it can significantly improve the performance. diff --git a/pkg/runtime/serializer/cbor/cbor.go b/pkg/runtime/serializer/cbor/cbor.go index 643d54d5a..46c1b0094 100644 --- a/pkg/runtime/serializer/cbor/cbor.go +++ b/pkg/runtime/serializer/cbor/cbor.go @@ -55,7 +55,14 @@ func (mf *defaultMetaFactory) Interpret(data []byte) (*schema.GroupVersionKind, type Serializer interface { runtime.Serializer + runtime.NondeterministicEncoder recognizer.RecognizingDecoder + + // NewSerializer returns a value of this interface type rather than exporting the serializer + // type and returning one of those because the zero value of serializer isn't ready to + // use. Users aren't intended to implement cbor.Serializer themselves, and this unexported + // interface method is here to prevent that (https://go.dev/blog/module-compatibility). + private() } var _ Serializer = &serializer{} @@ -79,6 +86,8 @@ type serializer struct { options options } +func (serializer) private() {} + func NewSerializer(creater runtime.ObjectCreater, typer runtime.ObjectTyper, options ...Option) Serializer { return newSerializer(&defaultMetaFactory{}, creater, typer, options...) } @@ -117,6 +126,10 @@ func (s *serializer) Encode(obj runtime.Object, w io.Writer) error { return s.encode(modes.Encode, obj, w) } +func (s *serializer) EncodeNondeterministic(obj runtime.Object, w io.Writer) error { + return s.encode(modes.EncodeNondeterministic, obj, w) +} + func (s *serializer) encode(mode modes.EncMode, obj runtime.Object, w io.Writer) error { var v interface{} = obj if u, ok := obj.(runtime.Unstructured); ok { diff --git a/pkg/runtime/serializer/cbor/cbor_test.go b/pkg/runtime/serializer/cbor/cbor_test.go index 8b8025ea8..a47858c6a 100644 --- a/pkg/runtime/serializer/cbor/cbor_test.go +++ b/pkg/runtime/serializer/cbor/cbor_test.go @@ -26,6 +26,7 @@ import ( "errors" "io" "reflect" + "strconv" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -762,3 +763,98 @@ type stubMetaFactory struct { func (mf stubMetaFactory) Interpret([]byte) (*schema.GroupVersionKind, error) { return mf.gvk, mf.err } + +type oneMapField struct { + metav1.TypeMeta `json:",inline"` + Map map[string]interface{} `json:"map"` +} + +func (o oneMapField) DeepCopyObject() runtime.Object { + panic("unimplemented") +} + +func (o oneMapField) GetObjectKind() schema.ObjectKind { + panic("unimplemented") +} + +type eightStringFields struct { + metav1.TypeMeta `json:",inline"` + A string `json:"1"` + B string `json:"2"` + C string `json:"3"` + D string `json:"4"` + E string `json:"5"` + F string `json:"6"` + G string `json:"7"` + H string `json:"8"` +} + +func (o eightStringFields) DeepCopyObject() runtime.Object { + panic("unimplemented") +} + +func (o eightStringFields) GetObjectKind() schema.ObjectKind { + panic("unimplemented") +} + +// TestEncodeNondeterministic tests that repeated encodings of multi-field structs and maps do not +// encode to precisely the same bytes when repeatedly encoded with EncodeNondeterministic. When +// using EncodeNondeterministic, the order of items in CBOR maps should be intentionally shuffled to +// prevent applications from inadvertently depending on encoding determinism. All permutations do +// not necessarily have equal probability. +func TestEncodeNondeterministic(t *testing.T) { + for _, tc := range []struct { + name string + input runtime.Object + }{ + { + name: "map", + input: func() runtime.Object { + m := map[string]interface{}{} + for i := 1; i <= 8; i++ { + m[strconv.Itoa(i)] = strconv.Itoa(i) + + } + return oneMapField{Map: m} + }(), + }, + { + name: "struct", + input: eightStringFields{ + TypeMeta: metav1.TypeMeta{}, + A: "1", + B: "2", + C: "3", + D: "4", + E: "5", + F: "6", + G: "7", + H: "8", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var b bytes.Buffer + e := NewSerializer(nil, nil) + + if err := e.EncodeNondeterministic(tc.input, &b); err != nil { + t.Fatal(err) + } + first := b.String() + + const Trials = 128 + for trial := 0; trial < Trials; trial++ { + b.Reset() + if err := e.EncodeNondeterministic(tc.input, &b); err != nil { + t.Fatal(err) + } + + if !bytes.Equal([]byte(first), b.Bytes()) { + return + } + } + t.Fatalf("nondeterministic encode produced the same bytes on %d consecutive calls: %s", Trials, first) + }) + } + +} diff --git a/pkg/runtime/serializer/cbor/internal/modes/encode.go b/pkg/runtime/serializer/cbor/internal/modes/encode.go index c66931384..5fae14151 100644 --- a/pkg/runtime/serializer/cbor/internal/modes/encode.go +++ b/pkg/runtime/serializer/cbor/internal/modes/encode.go @@ -105,7 +105,7 @@ var Encode = EncMode{ var EncodeNondeterministic = EncMode{ delegate: func() cbor.UserBufferEncMode { opts := Encode.options() - opts.Sort = cbor.SortNone // TODO: Use cbor.SortFastShuffle after bump to v2.7.0. + opts.Sort = cbor.SortFastShuffle em, err := opts.UserBufferEncMode() if err != nil { panic(err)