feat(schema): add API descriptors, struct, oneof & list types, and wi…

…re encoding spec (#21482)

Co-authored-by: marbar3778 <marbar3778@yahoo.com>
Co-authored-by: Marko <marko@baricevic.me>

schema/api.go

@@ -0,0 +1,98 @@
+package schema
+
+// APIDescriptor is a public versioned descriptor of an API.
+//
+// An APIDescriptor can be used as a native descriptor of an API's encoding.
+// The native binary encoding of API requests and responses is to encode the input and output
+// fields using value binary encoding.
+// The native JSON encoding would be to encode the fields as a JSON object, canonically
+// sorted by field name with no extra whitespace.
+// Thus, APIDefinitions have deterministic binary and JSON encodings.
+//
+// APIDefinitions have a strong definition of compatibility between different versions
+// of the same API.
+// It is an INCOMPATIBLE change to add new input fields to existing methods or to remove or modify
+// existing input or output fields.
+// Input fields also cannot reference any unsealed structs, directly or transitively,
+// because these types allow adding new fields.
+// Adding new input fields to a method introduces the possibility that a newer client
+// will send an incomprehensible message to an older server.
+// The only safe ways that input field schemas can be extended are by adding
+// new values to EnumType's and new cases to OneOfType's.
+// It is a COMPATIBLE change to add new methods to an API and to add new output fields
+// to existing methods.
+// Output fields can reference any sealed or unsealed StructType, directly or transitively.
+//
+// Existing protobuf APIs could also be mapped into APIDefinitions, and used in the following ways:
+// - to produce, user-friendly deterministic JSON
+// - to produce a deterministic binary encoding
+// - to check for compatibility in a way that is more appropriate to blockchain applications
+// - to use any code generators designed to support this spec as an alternative to protobuf
+// Also, a standardized way of serializing schema types as protobuf could be defined which
+// maps to the original protobuf encoding, so that schemas can be used as an interop
+// layer between different less expressive encoding systems.
+//
+// Existing EVM contract APIs expressed in Solidity could be mapped into APIDefinitions, and
+// a mapping of all schema values to ABI encoding could be defined which preserves the
+// original ABI encoding.
+//
+// In this way, we can define an interop layer between contracts in the EVM world,
+// SDK modules accepting protobuf types, and any API using this schema system natively.
+type APIDescriptor struct {
+	// Name is the versioned name of the API.
+	Name string
+
+	// Methods is the list of methods in the API.
+	// It is a COMPATIBLE change to add new methods to an API.
+	// If a newer client tries to call a method that an older server does not recognize it,
+	// an error will simply be returned.
+	Methods []MethodDescriptor
+}
+
+// MethodDescriptor describes a method in the API.
+type MethodDescriptor struct {
+	// Name is the name of the method.
+	Name string
+
+	// InputFields is the list of input fields for the method.
+	//
+	// It is an INCOMPATIBLE change to add, remove or update input fields to a method.
+	// The addition of new fields introduces the possibility that a newer client
+	// will send an incomprehensible message to an older server.
+	// InputFields can only reference sealed StructTypes, either directly and transitively.
+	//
+	// As a special case to represent protobuf service definitions, there can be a single
+	// unnamed struct input field that code generators can choose to either reference
+	// as a named struct or to expand inline as function arguments.
+	InputFields []Field
+
+	// OutputFields is the list of output fields for the method.
+	//
+	// It is a COMPATIBLE change to add new output fields to a method,
+	// but existing output fields should not be removed or modified.
+	// OutputFields can reference any sealed or unsealed StructType, directly or transitively.
+	// If a newer client tries to call a method on an older server, the newer expected result output
+	// fields will simply be populated with the default values for that field kind.
+	//
+	// As a special case to represent protobuf service definitions, there can be a single
+	// unnamed struct output field.
+	// In this case, adding new output fields is an INCOMPATIBLE change (because protobuf service definitions
+	// don't allow this), but new fields can be added to the referenced struct if it is unsealed.
+	OutputFields []Field
+
+	// Volatility is the volatility of the method.
+	Volatility Volatility
+}
+
+// Volatility is the volatility of a method.
+type Volatility int
+
+const (
+	// PureVolatility indicates that the method can neither read nor write state.
+	PureVolatility Volatility = iota
+	// ReadonlyVolatility indicates that the method can read state but not write state.
+	ReadonlyVolatility
+
+	// VolatileVolatility indicates that the method can read and write state.
+	VolatileVolatility
+)

schema/field.go

@@ -15,8 +15,21 @@ type Field struct {
 	// Nullable indicates whether null values are accepted for the field. Key fields CANNOT be nullable.
 	Nullable bool `json:"nullable,omitempty"`
 
-	// ReferencedType is the referenced type name when Kind is EnumKind.
+	// ReferencedType is the referenced type name when Kind is EnumKind, StructKind or OneOfKind.
 	ReferencedType string `json:"referenced_type,omitempty"`
+
+	// ElementKind is the element type when Kind is ListKind.
+	// Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added.
+	ElementKind Kind `json:"element_kind,omitempty"`
+
+	// Size specifies the size or max-size of a field.
+	// Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added.
+	// Its specific meaning may vary depending on the field kind.
+	// For IntNKind and UintNKind fields, it specifies the bit width of the field.
+	// For StringKind, BytesKind, AddressKind, and JSONKind, fields it specifies the maximum length rather than a fixed length.
+	// If it is 0, such fields have no maximum length.
+	// It is invalid to have a non-zero Size for other kinds.
+	Size uint32 `json:"size,omitempty"`
 }
 
 // Validate validates the field.

schema/kind.go

@@ -10,12 +10,25 @@ import (
 
 // Kind represents the basic type of a field in an object.
 // Each kind defines the following encodings:
-// Go Encoding: the golang type which should be accepted by listeners and
+//
+// - Go Encoding: the golang type which should be accepted by listeners and
 // generated by decoders when providing entity updates.
-// JSON Encoding: the JSON encoding which should be used when encoding the field to JSON.
+// - JSON Encoding: the JSON encoding which should be used when encoding the field to JSON.
+// - Key Binary Encoding: the encoding which should be used when encoding the field
+// as a key in binary messages. Some encodings specify a terminal and non-terminal form
+// depending on whether or not the field is the last field in the key.
+// - Value Binary Encoding: the encoding which should be used when encoding the field
+// as a value in binary messages.
+//
 // When there is some non-determinism in an encoding, kinds should specify what
 // values they accept and also what is the canonical, deterministic encoding which
 // should be preferably emitted by serializers.
+//
+// Binary encodings were chosen based on what is likely to be the most convenient default binary encoding
+// for state management implementations. This encoding allows for sorted keys whenever it is possible for a kind
+// and is deterministic.
+// Modules that use the specified encoding natively will have a trivial decoder implementation because the
+// encoding is already in the correct format after any initial prefix bytes are stripped.
 type Kind int
 
 const (
@@ -25,54 +38,82 @@ const (
 	// StringKind is a string type.
 	// Go Encoding: UTF-8 string with no null characters.
 	// JSON Encoding: string
+	// Key Binary Encoding:
+	//   non-terminal: UTF-8 string with no null characters suffixed with a null character
+	//   terminal: UTF-8 string with no null characters
+	// Value Binary Encoding: the same value binary encoding as BytesKind.
 	StringKind
 
-	// BytesKind is a bytes type.
+	// BytesKind represents a byte array.
 	// Go Encoding: []byte
 	// JSON Encoding: base64 encoded string, canonical values should be encoded with standard encoding and padding.
 	// Either standard or URL encoding with or without padding should be accepted.
+	// Key Binary Encoding:
+	//   non-terminal: length prefixed bytes where the width of the length prefix is 1, 2, 3 or 4 bytes depending on
+	//     the field's MaxLength (defaulting to 4 bytes).
+	//     Length prefixes should be big-endian encoded.
+	//     Values larger than 2^32 bytes are not supported (likely key-value stores impose a lower limit).
+	//   terminal: raw bytes with no length prefix
+	// Value Binary Encoding: two 32-bit unsigned little-endian integers, the first one representing the offset of the
+	//   value in the buffer and the second one representing the length of the value.
 	BytesKind
 
 	// Int8Kind represents an 8-bit signed integer.
 	// Go Encoding: int8
 	// JSON Encoding: number
+	// Key Binary Encoding: 1-byte two's complement encoding, with the first bit inverted for sorting.
+	// Value Binary Encoding: 1-byte two's complement encoding.
 	Int8Kind
 
 	// Uint8Kind represents an 8-bit unsigned integer.
 	// Go Encoding: uint8
 	// JSON Encoding: number
+	// Key Binary Encoding: 1-byte unsigned encoding.
+	// Value Binary Encoding: 1-byte unsigned encoding.
 	Uint8Kind
 
 	// Int16Kind represents a 16-bit signed integer.
 	// Go Encoding: int16
 	// JSON Encoding: number
+	// Key Binary Encoding: 2-byte two's complement big-endian encoding, with the first bit inverted for sorting.
+	// Value Binary Encoding: 2 byte two's complement little-endian encoding.
 	Int16Kind
 
 	// Uint16Kind represents a 16-bit unsigned integer.
 	// Go Encoding: uint16
 	// JSON Encoding: number
+	// Key Binary Encoding: 2-byte unsigned big-endian encoding.
+	// Value Binary Encoding: 2-byte unsigned little-endian encoding.
 	Uint16Kind
 
 	// Int32Kind represents a 32-bit signed integer.
 	// Go Encoding: int32
 	// JSON Encoding: number
+	// Key Binary Encoding: 4-byte two's complement big-endian encoding, with the first bit inverted for sorting.
+	// Value Binary Encoding: 4-byte two's complement little-endian encoding.
 	Int32Kind
 
 	// Uint32Kind represents a 32-bit unsigned integer.
 	// Go Encoding: uint32
 	// JSON Encoding: number
+	// Key Binary Encoding: 4-byte unsigned big-endian encoding.
+	// Value Binary Encoding: 4-byte unsigned little-endian encoding.
 	Uint32Kind
 
 	// Int64Kind represents a 64-bit signed integer.
 	// Go Encoding: int64
 	// JSON Encoding: base10 integer string which matches the IntegerFormat regex
 	// The canonical encoding should include no leading zeros.
+	// Key Binary Encoding: 8-byte two's complement big-endian encoding, with the first bit inverted for sorting.
+	// Value Binary Encoding: 8-byte two's complement little-endian encoding.
 	Int64Kind
 
 	// Uint64Kind represents a 64-bit unsigned integer.
 	// Go Encoding: uint64
 	// JSON Encoding: base10 integer string which matches the IntegerFormat regex
 	// Canonically encoded values should include no leading zeros.
+	// Key Binary Encoding: 8-byte unsigned big-endian encoding.
+	// Value Binary Encoding: 8-byte unsigned little-endian encoding.
 	Uint64Kind
 
 	// IntegerKind represents an arbitrary precision integer number.
@@ -98,6 +139,8 @@ const (
 	// BoolKind represents a boolean true or false value.
 	// Go Encoding: bool
 	// JSON Encoding: boolean
+	// Key Binary Encoding: 1-byte encoding where 0 is false and 1 is true.
+	// Value Binary Encoding: 1-byte encoding where 0 is false and 1 is true.
 	BoolKind
 
 	// TimeKind represents a nanosecond precision UNIX time value (with zero representing January 1, 1970 UTC).
@@ -107,44 +150,113 @@ const (
 	// Canonical values should be encoded with UTC time zone Z, nanoseconds should
 	// be encoded with no trailing zeros, and T time values should always be present
 	// even at 00:00:00.
+	// Key Binary Encoding: 8-byte two's complement big-endian encoding, with the first bit inverted for sorting.
+	// Value Binary Encoding: 8-byte two's complement little-endian encoding.
 	TimeKind
 
 	// DurationKind represents the elapsed time between two nanosecond precision time values.
 	// Its valid range is +/- 2^63 (the range of a 64-bit signed integer).
 	// Go Encoding: time.Duration
 	// JSON Encoding: the number of seconds as a decimal string with no trailing zeros followed by
 	// a lowercase 's' character to represent seconds.
+	// Key Binary Encoding: 8-byte two's complement big-endian encoding, with the first bit inverted for sorting.
+	// Value Binary Encoding: 8-byte two's complement little-endian encoding.
 	DurationKind
 
 	// Float32Kind represents an IEEE-754 32-bit floating point number.
 	// Go Encoding: float32
 	// JSON Encoding: number
+	// Key Binary Encoding: 4-byte IEEE-754 encoding.
+	// Value Binary Encoding: 4-byte IEEE-754 encoding.
 	Float32Kind
 
 	// Float64Kind represents an IEEE-754 64-bit floating point number.
 	// Go Encoding: float64
 	// JSON Encoding: number
+	// Key Binary Encoding: 8-byte IEEE-754 encoding.
+	// Value Binary Encoding: 8-byte IEEE-754 encoding.
 	Float64Kind
 
 	// AddressKind represents an account address which is represented by a variable length array of bytes.
 	// Addresses usually have a human-readable rendering, such as bech32, and tooling should provide
-	// a way for apps to define a string encoder for friendly user-facing display.
+	// a way for apps to define a string encoder for friendly user-facing display. Addresses have a maximum
+	// supported length of 63 bytes.
 	// Go Encoding: []byte
 	// JSON Encoding: addresses should be encoded as strings using the human-readable address renderer
 	// provided to the JSON encoder.
+	// Key Binary Encoding:
+	//   non-terminal: bytes prefixed with 1-byte length prefix
+	//   terminal: raw bytes with no length prefix
+	// Value Binary Encoding: bytes prefixed with 1-byte length prefix.
 	AddressKind
 
 	// EnumKind represents a value of an enum type.
 	// Fields of this type are expected to set the EnumType field in the field definition to the enum
 	// definition.
 	// Go Encoding: string
 	// JSON Encoding: string
+	// Key Binary Encoding: the same binary encoding as the EnumType's numeric kind.
+	// Value Binary Encoding: the same binary encoding as the EnumType's numeric kind.
 	EnumKind
 
 	// JSONKind represents arbitrary JSON data.
 	// Go Encoding: json.RawMessage
 	// JSON Encoding: any valid JSON value
+	// Key Binary Encoding: string encoding
+	// Value Binary Encoding: string encoding
 	JSONKind
+
+	// UIntNKind represents a signed integer type with a width in bits specified by the Size field in the
+	// field definition.
+	// Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added.
+	// N must be a multiple of 8, and it is invalid for N to equal 8, 16, 32, 64 as there are more specific
+	// types for these widths.
+	// Go Encoding: []byte where len([]byte) == Size / 8, little-endian encoded.
+	// JSON Encoding: base10 integer string matching the IntegerFormat regex, canonically with no leading zeros.
+	// Key Binary Encoding: N / 8 bytes big-endian encoded
+	// Value Binary Encoding: N / 8 bytes little-endian encoded
+	UIntNKind
+
+	// IntNKind represents an unsigned integer type with a width in bits specified by the Size field in the
+	// field definition. N must be a multiple of 8.
+	// Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added.
+	// N must be a multiple of 8, and it is invalid for N to equal 8, 16, 32, 64 as there are more specific
+	// types for these widths.
+	// Go Encoding: []byte where len([]byte) == Size / 8, two's complement little-endian encoded.
+	// JSON Encoding: base10 integer string matching the IntegerFormat regex, canonically with no leading zeros.
+	// Key Binary Encoding: N / 8 bytes big-endian two's complement encoded with the first bit inverted for sorting.
+	// Value Binary Encoding: N / 8 bytes little-endian two's complement encoded.
+	IntNKind
+
+	// StructKind represents a struct object.
+	// Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added.
+	// Go Encoding: an array of type []interface{} where each element is of the respective field's kind type.
+	// JSON Encoding: an object where each key is the field name and the value is the field value.
+	// Canonically, keys are in alphabetical order with no extra whitespace.
+	// Key Binary Encoding: not valid as a key field.
+	// Value Binary Encoding: 32-bit unsigned little-endian length prefix,
+	// followed by the value binary encoding of each field in order.
+	StructKind
+
+	// OneOfKind represents a field that can be one of a set of types.
+	// Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added.
+	// Go Encoding: the anonymous struct { Case string; Value interface{} }, aliased as OneOfValue.
+	// JSON Encoding: same as the case's struct encoding with "@type" set to the case name.
+	// Key Binary Encoding: not valid as a key field.
+	// Value Binary Encoding: the oneof's discriminant numeric value encoded as its discriminant kind
+	// followed by the encoded value.
+	OneOfKind
+
+	// ListKind represents a list of elements.
+	// Support for this is currently UNIMPLEMENTED, this notice will be removed when it is added.
+	// Go Encoding: an array of type []interface{} where each element is of the respective field's kind type.
+	// JSON Encoding: an array of values where each element is the field value.
+	// Canonically, there is no extra whitespace.
+	// Key Binary Encoding: not valid as a key field.
+	// Value Binary Encoding: 32-bit unsigned little-endian size prefix indicating the size of the encoded data in bytes,
+	// followed by a 32-bit unsigned little-endian count of the number of elements in the list,
+	// followed by each element encoded with value binary encoding.
+	ListKind
 )
 
 // MAX_VALID_KIND is the maximum valid kind value.