Avoid intermediate JSON object allocations when generating JSON strings

benchmarks/bin/to_proto3_json_string.dart

@@ -2,8 +2,6 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'dart:convert' show jsonEncode;
-
 import 'package:protobuf_benchmarks/benchmark_base.dart';
 import 'package:protobuf_benchmarks/generated/google_message1_proto2.pb.dart'
  as p2;
@@ -26,9 +24,9 @@ class Benchmark extends BenchmarkBase {
 
  @override
  void run() {
- jsonEncode(_message1Proto2.toProto3Json());
- jsonEncode(_message1Proto3.toProto3Json());
- jsonEncode(_message2.toProto3Json());
+ _message1Proto2.toProto3JsonString();
+ _message1Proto3.toProto3JsonString();
+ _message2.toProto3JsonString();
  }
 }
 

benchmarks/pubspec.lock

@@ -43,6 +43,13 @@ packages:
  url: "https://pub.dartlang.org"
  source: hosted
  version: "0.6.4"
+ jsontool:
+ dependency: transitive
+ description:
+ name: jsontool
+ url: "https://pub.dartlang.org"
+ source: hosted
+ version: "1.1.2"
  lints:
  dependency: "direct dev"
  description:

protobuf/CHANGELOG.md

@@ -20,6 +20,20 @@
  missing. ([#719], [#745])
 * Fix updating frozen (immutable) messages with merge methods
  (`mergeFromBuffer`, `mergeFromProto3Json`, ...). ([#489], [#727])
+* 3 new `GeneratedMessage` methods added for JSON serialization:
+
+ - `toProto3JsonSink`: for serializing a message as proto3 JSON to a
+ [jsontool] sink
+ - `toProto3JsonString`: for serializing a message as proto3 JSON string
+ - `writeToJsonSink`: for serializing a message as custom JSON format (the
+ format used by `writeToJsonMap`) to a [jsontool] sink
+
+ Methods that return JSON strings (`writeToJson`, `toProto3JsonString`) now
+ don't allocate intermediate JSON objects and are much more efficient than
+ generating JSON objects (`writeToJsonMap`, `toProto3Json`) and then encoding
+ with `dart:convert`'s `json.encode`.
+
+ ([#683])
 
 [#183]: https://github.com/google/protobuf.dart/issues/183
 [#644]: https://github.com/google/protobuf.dart/pull/644
@@ -37,6 +51,8 @@
 [#745]: https://github.com/google/protobuf.dart/pull/745
 [#489]: https://github.com/google/protobuf.dart/issues/489
 [#727]: https://github.com/google/protobuf.dart/pull/727
+[jsontool]: https://pub.dev/packages/jsontool
+[#683]: https://github.com/google/protobuf.dart/pull/683
 
 ## 2.1.0
 

protobuf/lib/meta.dart

@@ -52,12 +52,15 @@ const GeneratedMessage_reservedNames = <String>[
  'toBuilder',
  'toDebugString',
  'toProto3Json',
+ 'toProto3JsonSink',
+ 'toProto3JsonString',
  'toString',
  'unknownFields',
  'writeToBuffer',
  'writeToCodedBufferWriter',
  'writeToJson',
  'writeToJsonMap',
+ 'writeToJsonSink',
  r'$_ensure',
  r'$_get',
  r'$_getI64',

protobuf/lib/protobuf.dart

@@ -14,6 +14,7 @@ import 'dart:math' as math;
 import 'dart:typed_data' show TypedData, Uint8List, ByteData, Endian;
 
 import 'package:fixnum/fixnum.dart' show Int64;
+import 'package:jsontool/jsontool.dart';
 import 'package:meta/meta.dart' show UseResult;
 
 import 'src/protobuf/json_parsing_context.dart';
@@ -38,11 +39,13 @@ part 'src/protobuf/field_set.dart';
 part 'src/protobuf/field_type.dart';
 part 'src/protobuf/generated_message.dart';
 part 'src/protobuf/generated_service.dart';
-part 'src/protobuf/json.dart';
+part 'src/protobuf/json_reader.dart';
+part 'src/protobuf/json_writer.dart';
 part 'src/protobuf/pb_list.dart';
 part 'src/protobuf/pb_map.dart';
 part 'src/protobuf/protobuf_enum.dart';
-part 'src/protobuf/proto3_json.dart';
+part 'src/protobuf/proto3_json_reader.dart';
+part 'src/protobuf/proto3_json_writer.dart';
 part 'src/protobuf/rpc_client.dart';
 part 'src/protobuf/unknown_field_set.dart';
 part 'src/protobuf/utils.dart';

protobuf/lib/src/protobuf/builder_info.dart

@@ -41,11 +41,15 @@ class BuilderInfo {
 
  List<FieldInfo>? _sortedByTag;
 
- // For well-known types.
- final Object? Function(GeneratedMessage message, TypeRegistry typeRegistry)?
+ /// JSON generator for well-known types.
+ final void Function(
+ GeneratedMessage msg, TypeRegistry typeRegistry, JsonSink jsonSink)?
  toProto3Json;
+
+ /// JSON parser for well-known types.
  final Function(GeneratedMessage targetMessage, Object json,
  TypeRegistry typeRegistry, JsonParsingContext context)? fromProto3Json;
+
  final CreateBuilderFunc? createEmptyInstance;
 
  BuilderInfo(String? messageName,

protobuf/lib/src/protobuf/generated_message.dart

@@ -200,7 +200,14 @@ abstract class GeneratedMessage {
  /// Returns the JSON encoding of this message as a Dart [Map].
  ///
  /// The encoding is described in [GeneratedMessage.writeToJson].
- Map<String, dynamic> writeToJsonMap() => _writeToJsonMap(_fieldSet);
+ Map<String, dynamic> writeToJsonMap() {
+ Object? object;
+ final objectSink = jsonObjectWriter((newObject) {
+ object = newObject;
+ });
+ writeToJsonSink(objectSink);
+ return object as Map<String, dynamic>;
+ }
 
  /// Returns a JSON string that encodes this message.
  ///
@@ -215,24 +222,65 @@ abstract class GeneratedMessage {
  /// represented as their integer value.
  ///
  /// For the proto3 JSON format use: [toProto3Json].
- String writeToJson() => jsonEncode(writeToJsonMap());
+ String writeToJson() {
+ final buf = StringBuffer();
+ final stringSink = jsonStringWriter(buf);
+ writeToJsonSink(stringSink);
+ return buf.toString();
+ }
+
+ void writeToJsonSink(JsonSink jsonSink) {
+ _writeToJsonMapSink(_fieldSet, jsonSink);
+ }
+
+ /// Returns Dart JSON object encoding this message following proto3 JSON
+ /// format.
+ ///
+ /// The returned object will have the same format as objects returned by
+ /// [jsonEncode].
+ ///
+ /// See [toProto3JsonSink] for details.
+ Object? toProto3Json(
+ {TypeRegistry typeRegistry = const TypeRegistry.empty()}) {
+ Object? object;
+ final objectSink = jsonObjectWriter((newObject) {
+ object = newObject;
+ });
+ _writeToProto3JsonSink(_fieldSet, typeRegistry, objectSink);
+ return object;
+ }
 
- /// Returns an Object representing Proto3 JSON serialization of `this`.
+ /// Returns a proto3 JSON string encoding this message.
+ ///
+ /// See [toProto3JsonSink] for details.
+ String toProto3JsonString(
+ {TypeRegistry typeRegistry = const TypeRegistry.empty()}) {
+ final buf = StringBuffer();
+ final stringSink = jsonStringWriter(buf);
+ toProto3JsonSink(typeRegistry, stringSink);
+ return buf.toString();
+ }
+
+ /// Writes proto3 JSON serialization of this message to the given [JsonSink].
  ///
  /// The key for each field is be the camel-cased name of the field.
  ///
- /// Well-known types and their special JSON encoding are supported.
- /// If a well-known type cannot be encoded (eg. a `google.protobuf.Timestamp`
- /// with negative `nanoseconds`) an error is thrown.
+ /// Well-known types and their special JSON encoding are supported. If a
+ /// well-known type cannot be encoded (eg. a `google.protobuf.Timestamp` with
+ /// negative `nanoseconds`) an error is thrown.
  ///
  /// Extensions and unknown fields are not encoded.
  ///
  /// The [typeRegistry] is be used for encoding `Any` messages. If an `Any`
- /// message encoding a type not in [typeRegistry] is encountered, an
- /// error is thrown.
- Object? toProto3Json(
- {TypeRegistry typeRegistry = const TypeRegistry.empty()}) =>
- _writeToProto3Json(_fieldSet, typeRegistry);
+ /// message encoding a type not in [typeRegistry] is encountered, an error is
+ /// thrown.
+ ///
+ /// The [newMessage] argument is for use in generated code, do not use.
+ void toProto3JsonSink(TypeRegistry typeRegistry, JsonSink jsonSink,
+ {bool newMessage = true}) {
+ _writeToProto3JsonSink(_fieldSet, typeRegistry, jsonSink,
+ newMessage: newMessage);
+ }
 
  /// Merges field values from [json], a JSON object using proto3 encoding.
  ///

protobuf/lib/src/protobuf/json.dart → protobuf/lib/src/protobuf/json_reader.dart

@@ -4,94 +4,6 @@
 
 part of protobuf;
 
-Map<String, dynamic> _writeToJsonMap(_FieldSet fs) {
- dynamic convertToMap(dynamic fieldValue, int fieldType) {
- var baseType = PbFieldType._baseType(fieldType);
-
- if (_isRepeated(fieldType)) {
- final PbList list = fieldValue;
- return List.from(list.map((e) => convertToMap(e, baseType)));
- }
-
- switch (baseType) {
- case PbFieldType._BOOL_BIT:
- case PbFieldType._STRING_BIT:
- case PbFieldType._INT32_BIT:
- case PbFieldType._SINT32_BIT:
- case PbFieldType._UINT32_BIT:
- case PbFieldType._FIXED32_BIT:
- case PbFieldType._SFIXED32_BIT:
- return fieldValue;
- case PbFieldType._FLOAT_BIT:
- case PbFieldType._DOUBLE_BIT:
- final value = fieldValue as double;
- if (value.isNaN) {
- return _nan;
- }
- if (value.isInfinite) {
- return value.isNegative ? _negativeInfinity : _infinity;
- }
- if (fieldValue.toInt() == fieldValue) {
- return fieldValue.toInt();
- }
- return value;
- case PbFieldType._BYTES_BIT:
- // Encode 'bytes' as a base64-encoded string.
- return base64Encode(fieldValue as List<int>);
- case PbFieldType._ENUM_BIT:
- final ProtobufEnum enum_ = fieldValue;
- return enum_.value; // assume |value| < 2^52
- case PbFieldType._INT64_BIT:
- case PbFieldType._SINT64_BIT:
- case PbFieldType._SFIXED64_BIT:
- return fieldValue.toString();
- case PbFieldType._UINT64_BIT:
- case PbFieldType._FIXED64_BIT:
- final Int64 int_ = fieldValue;
- return int_.toStringUnsigned();
- case PbFieldType._GROUP_BIT:
- case PbFieldType._MESSAGE_BIT:
- final GeneratedMessage msg = fieldValue;
- return msg.writeToJsonMap();
- default:
- throw 'Unknown type $fieldType';
- }
- }
-
- List _writeMap(PbMap fieldValue, MapFieldInfo fi) =>
- List.from(fieldValue.entries.map((MapEntry e) => {
- '${PbMap._keyFieldNumber}': convertToMap(e.key, fi.keyFieldType),
- '${PbMap._valueFieldNumber}':
- convertToMap(e.value, fi.valueFieldType)
- }));
-
- var result = <String, dynamic>{};
- for (var fi in fs._infosSortedByTag) {
- var value = fs._values[fi.index!];
- if (value == null || (value is List && value.isEmpty)) {
- continue; // It's missing, repeated, or an empty byte array.
- }
- if (_isMapField(fi.type)) {
- result['${fi.tagNumber}'] =
- _writeMap(value, fi as MapFieldInfo<dynamic, dynamic>);
- continue;
- }
- result['${fi.tagNumber}'] = convertToMap(value, fi.type);
- }
- final extensions = fs._extensions;
- if (extensions != null) {
- for (var tagNumber in _sorted(extensions._tagNumbers)) {
- var value = extensions._values[tagNumber];
- if (value is List && value.isEmpty) {
- continue; // It's repeated or an empty byte array.
- }
- var fi = extensions._getInfoOrNull(tagNumber)!;
- result['$tagNumber'] = convertToMap(value, fi.type);
- }
- }
- return result;
-}
-
 // Merge fields from a previously decoded JSON object.
 // (Called recursively on nested messages.)
 void _mergeFromJsonMap(
@@ -146,14 +58,14 @@ void _appendJsonMap(BuilderInfo meta, _FieldSet fs, List jsonList,
  final convertedKey = _convertJsonValue(
  entryMeta,
  entryFieldSet,
- jsonEntry['${PbMap._keyFieldNumber}'],
+ jsonEntry[PbMap._keyFieldNumberString],
  PbMap._keyFieldNumber,
  fi.keyFieldType,
  registry);
  var convertedValue = _convertJsonValue(
  entryMeta,
  entryFieldSet,
- jsonEntry['${PbMap._valueFieldNumber}'],
+ jsonEntry[PbMap._valueFieldNumberString],
  PbMap._valueFieldNumber,
  fi.valueFieldType,
  registry);