diff --git a/CHANGES.txt b/CHANGES.txt index 3c6bc35107a05..ffb9c08a2b0b3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -15,6 +15,7 @@ Java * Update protobuf_version.bzl to separate protoc and per-language java … (#9900) * 6x speedup in ArrayEncoder.writeUInt32NotTag + * Java generated code is no longer compatible with runtimes 2.6.1 and earlier Python * Increment python major version to 4 in version.json for python upb (#9926) @@ -26,6 +27,7 @@ * Due to the breaking changes for Python, the major version number for Python has been incremented. * The binary wheel for macOS now supports Apple silicon. + * In TextFormat, transform UnicodeDecodeError into ParseError. PHP diff --git a/csharp/src/Google.Protobuf.Test/testprotos.pb b/csharp/src/Google.Protobuf.Test/testprotos.pb index 83958c6097de2..4bc182570b31f 100644 Binary files a/csharp/src/Google.Protobuf.Test/testprotos.pb and b/csharp/src/Google.Protobuf.Test/testprotos.pb differ diff --git a/csharp/src/Google.Protobuf/Reflection/Descriptor.cs b/csharp/src/Google.Protobuf/Reflection/Descriptor.cs index 1cb43c0ea38cb..7e7a1a528e7f0 100644 --- a/csharp/src/Google.Protobuf/Reflection/Descriptor.cs +++ b/csharp/src/Google.Protobuf/Reflection/Descriptor.cs @@ -7098,11 +7098,8 @@ public void ClearJstype() { /// check its required fields, regardless of whether or not the message has /// been parsed. /// - /// As of 2021, lazy does no correctness checks on the byte stream during - /// parsing. This may lead to crashes if and when an invalid byte stream is - /// finally parsed upon access. - /// - /// TODO(b/211906113): Enable validation on lazy fields. + /// As of May 2022, lazy verifies the contents of the byte stream during + /// parsing. An invalid byte stream will cause the overall parsing to fail. /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index a36195bbb3755..07d7715820d71 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -461,21 +461,20 @@ public static FileDescriptor internalBuildGeneratedFileFrom( } /** - * This method is to be called by generated code only. It is used to update the + * This method is to be called by generated code only. It updates the * FileDescriptorProto associated with the descriptor by parsing it again with the given * ExtensionRegistry. This is needed to recognize custom options. */ public static void internalUpdateFileDescriptor( - final FileDescriptor descriptor, final ExtensionRegistry registry) { + FileDescriptor descriptor, ExtensionRegistry registry) { ByteString bytes = descriptor.proto.toByteString(); - FileDescriptorProto proto; try { - proto = FileDescriptorProto.parseFrom(bytes, registry); + FileDescriptorProto proto = FileDescriptorProto.parseFrom(bytes, registry); + descriptor.setProto(proto); } catch (InvalidProtocolBufferException e) { throw new IllegalArgumentException( "Failed to parse protocol buffer descriptor for generated code.", e); } - descriptor.setProto(proto); } /** diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java index 9db33456d6344..e212ab5fe0993 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java @@ -244,11 +244,16 @@ public static enum MethodToInvoke { * *

For use by generated code only. */ - protected abstract Object dynamicMethod(MethodToInvoke method, Object arg0, Object arg1); + protected abstract Object dynamicMethod( + MethodToInvoke method, + Object arg0, + Object arg1); /** Same as {@link #dynamicMethod(MethodToInvoke, Object, Object)} with {@code null} padding. */ @CanIgnoreReturnValue - protected Object dynamicMethod(MethodToInvoke method, Object arg0) { + protected Object dynamicMethod( + MethodToInvoke method, + Object arg0) { return dynamicMethod(method, arg0, null); } @@ -1245,11 +1250,11 @@ public MessageLite getMessageDefaultInstance() { } @SuppressWarnings("unchecked") - Object fromFieldSetType(final Object value) { + Object fromFieldSetType(Object value) { if (descriptor.isRepeated()) { if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) { - final List result = new ArrayList<>(); - for (final Object element : (List) value) { + List result = new ArrayList<>(); + for (Object element : (List) value) { result.add(singularFromFieldSetType(element)); } return result; @@ -1261,7 +1266,7 @@ Object fromFieldSetType(final Object value) { } } - Object singularFromFieldSetType(final Object value) { + Object singularFromFieldSetType(Object value) { if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) { return descriptor.enumTypeMap.findValueByNumber((Integer) value); } else { @@ -1269,12 +1274,11 @@ Object singularFromFieldSetType(final Object value) { } } - @SuppressWarnings("unchecked") - Object toFieldSetType(final Object value) { + Object toFieldSetType(Object value) { if (descriptor.isRepeated()) { if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) { - final List result = new ArrayList<>(); - for (final Object element : (List) value) { + List result = new ArrayList<>(); + for (Object element : (List) value) { result.add(singularToFieldSetType(element)); } return result; @@ -1286,7 +1290,7 @@ Object toFieldSetType(final Object value) { } } - Object singularToFieldSetType(final Object value) { + Object singularToFieldSetType(Object value) { if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) { return ((Internal.EnumLite) value).getNumber(); } else { diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index 7317f49ffeb3d..7fba30a04ea2f 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -116,8 +116,8 @@ public static void printUnicode(final UnknownFieldSet fields, final Appendable o /** * Generates a human readable form of this message, useful for debugging and other purposes, with - * no newline characters. This is just a trivial wrapper around - * {@link TextFormat.Printer#shortDebugString(MessageOrBuilder)}. + * no newline characters. This is just a trivial wrapper around {@link + * TextFormat.Printer#shortDebugString(MessageOrBuilder)}. */ public static String shortDebugString(final MessageOrBuilder message) { return printer().shortDebugString(message); @@ -459,9 +459,7 @@ private void printField( } } - /** - * An adapter class that can take a {@link MapEntry} and returns its key and entry. - */ + /** An adapter class that can take a {@link MapEntry} and returns its key and entry. */ private static class MapEntryAdapter implements Comparable { private Object entry; @@ -953,6 +951,7 @@ private static final class Tokenizer { * the next token is parsed. */ private boolean containsSilentMarkerAfterCurrentToken = false; + private boolean containsSilentMarkerAfterPrevToken = false; /** Construct a tokenizer that parses tokens from the given text. */ @@ -1378,7 +1377,6 @@ private ParseException integerParseException(final NumberFormatException e) { private ParseException floatParseException(final NumberFormatException e) { return parseException("Couldn't parse number: " + e.getMessage()); } - } /** Thrown when parsing an invalid text format message. */ @@ -1551,7 +1549,7 @@ public static class Parser { * the current token is part of the field value, so the silent marker is indicated by * containsSilentMarkerAfterPrevToken. */ - private void detectSilentMarker(Tokenizer tokenizer) { + private void detectSilentMarker(Tokenizer tokenizer, String fieldName) { } /** @@ -1628,8 +1626,8 @@ public Builder setTypeRegistry(TypeRegistry typeRegistry) { * unknown field is encountered. If this is set, the parser will only log a warning. Allow * unknown fields will also allow unknown extensions. * - *

Use of this parameter is discouraged which may hide some errors (e.g. - * spelling error on field name). + *

Use of this parameter is discouraged which may hide some errors (e.g. spelling error on + * field name). */ public Builder setAllowUnknownFields(boolean allowUnknownFields) { this.allowUnknownFields = allowUnknownFields; @@ -1637,10 +1635,9 @@ public Builder setAllowUnknownFields(boolean allowUnknownFields) { } /** - * Set whether this parser will allow unknown extensions. By default, an - * exception is thrown if unknown extension is encountered. If this is set true, - * the parser will only log a warning. Allow unknown extensions does not mean - * allow normal unknown fields. + * Set whether this parser will allow unknown extensions. By default, an exception is thrown + * if unknown extension is encountered. If this is set true, the parser will only log a + * warning. Allow unknown extensions does not mean allow normal unknown fields. */ public Builder setAllowUnknownExtensions(boolean allowUnknownExtensions) { this.allowUnknownExtensions = allowUnknownExtensions; @@ -1725,7 +1722,8 @@ private static StringBuilder toStringBuilder(final Readable input) throws IOExce static final class UnknownField { static enum Type { - FIELD, EXTENSION; + FIELD, + EXTENSION; } final String message; @@ -1786,7 +1784,6 @@ public void merge( throws ParseException { final Tokenizer tokenizer = new Tokenizer(input); MessageReflection.BuilderAdapter target = new MessageReflection.BuilderAdapter(builder); - List unknownFields = new ArrayList(); while (!tokenizer.atEnd()) { @@ -1803,12 +1800,7 @@ private void mergeField( final MessageReflection.MergeTarget target, List unknownFields) throws ParseException { - mergeField( - tokenizer, - extensionRegistry, - target, - parseInfoTreeBuilder, - unknownFields); + mergeField(tokenizer, extensionRegistry, target, parseInfoTreeBuilder, unknownFields); } /** Parse a single field from {@code tokenizer} and merge it into {@code target}. */ @@ -1820,26 +1812,28 @@ private void mergeField( List unknownFields) throws ParseException { FieldDescriptor field = null; + String name; int startLine = tokenizer.getLine(); int startColumn = tokenizer.getColumn(); final Descriptor type = target.getDescriptorForType(); ExtensionRegistry.ExtensionInfo extension = null; if ("google.protobuf.Any".equals(type.getFullName()) && tokenizer.tryConsume("[")) { - mergeAnyFieldValue(tokenizer, extensionRegistry, target, parseTreeBuilder, unknownFields, - type); + mergeAnyFieldValue( + tokenizer, extensionRegistry, target, parseTreeBuilder, unknownFields, type); return; } if (tokenizer.tryConsume("[")) { // An extension. - final StringBuilder name = new StringBuilder(tokenizer.consumeIdentifier()); + StringBuilder nameBuilder = new StringBuilder(tokenizer.consumeIdentifier()); while (tokenizer.tryConsume(".")) { - name.append('.'); - name.append(tokenizer.consumeIdentifier()); + nameBuilder.append('.'); + nameBuilder.append(tokenizer.consumeIdentifier()); } + name = nameBuilder.toString(); - extension = target.findExtensionByName(extensionRegistry, name.toString()); + extension = target.findExtensionByName(extensionRegistry, name); if (extension == null) { String message = @@ -1866,7 +1860,7 @@ private void mergeField( tokenizer.consume("]"); } else { - final String name = tokenizer.consumeIdentifier(); + name = tokenizer.consumeIdentifier(); field = type.findFieldByName(name); // Group names are expected to be capitalized as they appear in the @@ -1890,13 +1884,14 @@ private void mergeField( } if (field == null) { - String message = (tokenizer.getPreviousLine() + 1) - + ":" - + (tokenizer.getPreviousColumn() + 1) - + ":\t" - + type.getFullName() - + "." - + name; + String message = + (tokenizer.getPreviousLine() + 1) + + ":" + + (tokenizer.getPreviousColumn() + 1) + + ":\t" + + type.getFullName() + + "." + + name; unknownFields.add(new UnknownField(message, UnknownField.Type.FIELD)); } } @@ -1909,7 +1904,7 @@ private void mergeField( // start with "{" or "<" which indicates the beginning of a message body. // If there is no ":" or there is a "{" or "<" after ":", this field has // to be a message or the input is ill-formed. - detectSilentMarker(tokenizer); + detectSilentMarker(tokenizer, name); if (tokenizer.tryConsume(":") && !tokenizer.lookingAt("{") && !tokenizer.lookingAt("<")) { skipFieldValue(tokenizer); } else { @@ -1920,7 +1915,7 @@ private void mergeField( // Handle potential ':'. if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { - detectSilentMarker(tokenizer); + detectSilentMarker(tokenizer, field.getFullName()); tokenizer.tryConsume(":"); // optional if (parseTreeBuilder != null) { TextFormatParseInfoTree.Builder childParseTreeBuilder = @@ -1944,7 +1939,7 @@ private void mergeField( unknownFields); } } else { - detectSilentMarker(tokenizer); + detectSilentMarker(tokenizer, field.getFullName()); tokenizer.consume(":"); // required consumeFieldValues( tokenizer, @@ -1967,6 +1962,29 @@ private void mergeField( } } + private String consumeFullTypeName(Tokenizer tokenizer) throws ParseException { + // If there is not a leading `[`, this is just a type name. + if (!tokenizer.tryConsume("[")) { + return tokenizer.consumeIdentifier(); + } + + // Otherwise, this is an extension or google.protobuf.Any type URL: we consume proto path + // elements until we've addressed the type. + String name = tokenizer.consumeIdentifier(); + while (tokenizer.tryConsume(".")) { + name += "." + tokenizer.consumeIdentifier(); + } + if (tokenizer.tryConsume("/")) { + name += "/" + tokenizer.consumeIdentifier(); + while (tokenizer.tryConsume(".")) { + name += "." + tokenizer.consumeIdentifier(); + } + } + tokenizer.consume("]"); + + return name; + } + /** * Parse a one or more field values from {@code tokenizer} and merge it into {@code builder}. */ @@ -2058,8 +2076,13 @@ private void consumeFieldValue( // (java_proto_library for any_java_proto depends on the protobuf_impl). Message anyBuilder = DynamicMessage.getDefaultInstance(field.getMessageType()); MessageReflection.MergeTarget anyField = target.newMergeTargetForField(field, anyBuilder); - mergeAnyFieldValue(tokenizer, extensionRegistry, anyField, parseTreeBuilder, - unknownFields, field.getMessageType()); + mergeAnyFieldValue( + tokenizer, + extensionRegistry, + anyField, + parseTreeBuilder, + unknownFields, + field.getMessageType()); value = anyField.finish(); tokenizer.consume(endToken); } else { @@ -2206,7 +2229,7 @@ private void mergeAnyFieldValue( throw tokenizer.parseExceptionPreviousToken("Expected a valid type URL."); } } - detectSilentMarker(tokenizer); + detectSilentMarker(tokenizer, typeUrlBuilder.toString()); tokenizer.tryConsume(":"); final String anyEndToken; if (tokenizer.tryConsume("<")) { @@ -2244,15 +2267,7 @@ private void mergeAnyFieldValue( /** Skips the next field including the field's name and value. */ private void skipField(Tokenizer tokenizer) throws ParseException { - if (tokenizer.tryConsume("[")) { - // Extension name. - do { - tokenizer.consumeIdentifier(); - } while (tokenizer.tryConsume(".")); - tokenizer.consume("]"); - } else { - tokenizer.consumeIdentifier(); - } + String name = consumeFullTypeName(tokenizer); // Try to guess the type of this field. // If this field is not a message, there should be a ":" between the @@ -2260,7 +2275,7 @@ private void skipField(Tokenizer tokenizer) throws ParseException { // start with "{" or "<" which indicates the beginning of a message body. // If there is no ":" or there is a "{" or "<" after ":", this field has // to be a message or the input is ill-formed. - detectSilentMarker(tokenizer); + detectSilentMarker(tokenizer, name); if (tokenizer.tryConsume(":") && !tokenizer.lookingAt("<") && !tokenizer.lookingAt("{")) { skipFieldValue(tokenizer); } else { @@ -2469,9 +2484,10 @@ && isHex(input.byteAt(i + 3))) { } Character.UnicodeBlock unicodeBlock = Character.UnicodeBlock.of(codepoint); if (unicodeBlock != null - && (unicodeBlock.equals(Character.UnicodeBlock.LOW_SURROGATES) - || unicodeBlock.equals(Character.UnicodeBlock.HIGH_SURROGATES) - || unicodeBlock.equals(Character.UnicodeBlock.HIGH_PRIVATE_USE_SURROGATES))) { + && (unicodeBlock.equals(Character.UnicodeBlock.LOW_SURROGATES) + || unicodeBlock.equals(Character.UnicodeBlock.HIGH_SURROGATES) + || unicodeBlock.equals( + Character.UnicodeBlock.HIGH_PRIVATE_USE_SURROGATES))) { throw new InvalidEscapeSequenceException( "Invalid escape sequence: '\\U" + input.substring(i, i + 8).toStringUtf8() diff --git a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java index cde17768309b1..1689c72e978df 100644 --- a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java +++ b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java @@ -67,9 +67,7 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** - * Test case for {@link TextFormat}. - */ +/** Test case for {@link TextFormat}. */ @RunWith(JUnit4.class) public class TextFormatTest { @@ -1450,6 +1448,18 @@ public void testParseUnknownExtensions() throws Exception { + "unknown_field3: 3\n"); } + @Test + public void testParseUnknownExtensionWithAnyMessage() throws Exception { + assertParseSuccessWithUnknownExtensions( + "[unknown_extension]: { " + + " any_value { " + + " [type.googleapis.com/protobuf_unittest.OneString] { " + + " data: 123 " + + " } " + + " } " + + "}"); + } + // See additional coverage in testOneofOverwriteForbidden and testMapOverwriteForbidden. @Test public void testParseNonRepeatedFields() throws Exception { diff --git a/php/src/Google/Protobuf/Internal/FieldOptions.php b/php/src/Google/Protobuf/Internal/FieldOptions.php index 665cb065a80db..ea32e7f2adf69 100644 --- a/php/src/Google/Protobuf/Internal/FieldOptions.php +++ b/php/src/Google/Protobuf/Internal/FieldOptions.php @@ -74,10 +74,8 @@ class FieldOptions extends \Google\Protobuf\Internal\Message * implementation must either *always* check its required fields, or *never* * check its required fields, regardless of whether or not the message has * been parsed. - * As of 2021, lazy does no correctness checks on the byte stream during - * parsing. This may lead to crashes if and when an invalid byte stream is - * finally parsed upon access. - * TODO(b/211906113): Enable validation on lazy fields. + * As of May 2022, lazy verifies the contents of the byte stream during + * parsing. An invalid byte stream will cause the overall parsing to fail. * * Generated from protobuf field optional bool lazy = 5 [default = false]; */ @@ -165,10 +163,8 @@ class FieldOptions extends \Google\Protobuf\Internal\Message * implementation must either *always* check its required fields, or *never* * check its required fields, regardless of whether or not the message has * been parsed. - * As of 2021, lazy does no correctness checks on the byte stream during - * parsing. This may lead to crashes if and when an invalid byte stream is - * finally parsed upon access. - * TODO(b/211906113): Enable validation on lazy fields. + * As of May 2022, lazy verifies the contents of the byte stream during + * parsing. An invalid byte stream will cause the overall parsing to fail. * @type bool $unverified_lazy * unverified_lazy does no correctness checks on the byte stream. This should * only be used where lazy with verification is prohibitive for performance @@ -354,10 +350,8 @@ public function setJstype($var) * implementation must either *always* check its required fields, or *never* * check its required fields, regardless of whether or not the message has * been parsed. - * As of 2021, lazy does no correctness checks on the byte stream during - * parsing. This may lead to crashes if and when an invalid byte stream is - * finally parsed upon access. - * TODO(b/211906113): Enable validation on lazy fields. + * As of May 2022, lazy verifies the contents of the byte stream during + * parsing. An invalid byte stream will cause the overall parsing to fail. * * Generated from protobuf field optional bool lazy = 5 [default = false]; * @return bool @@ -402,10 +396,8 @@ public function clearLazy() * implementation must either *always* check its required fields, or *never* * check its required fields, regardless of whether or not the message has * been parsed. - * As of 2021, lazy does no correctness checks on the byte stream during - * parsing. This may lead to crashes if and when an invalid byte stream is - * finally parsed upon access. - * TODO(b/211906113): Enable validation on lazy fields. + * As of May 2022, lazy verifies the contents of the byte stream during + * parsing. An invalid byte stream will cause the overall parsing to fail. * * Generated from protobuf field optional bool lazy = 5 [default = false]; * @param bool $var diff --git a/python/google/protobuf/descriptor_pool.py b/python/google/protobuf/descriptor_pool.py index 911372a8b00e5..7f84a63feab62 100644 --- a/python/google/protobuf/descriptor_pool.py +++ b/python/google/protobuf/descriptor_pool.py @@ -144,9 +144,6 @@ def __init__(self, descriptor_db=None): self._service_descriptors = {} self._file_descriptors = {} self._toplevel_extensions = {} - # TODO(jieluo): Remove _file_desc_by_toplevel_extension after - # maybe year 2020 for compatibility issue (with 3.4.1 only). - self._file_desc_by_toplevel_extension = {} self._top_enum_values = {} # We store extensions in two two-level mappings: The first key is the # descriptor of the message being extended, the second key is the extension @@ -331,6 +328,8 @@ def _AddExtensionDescriptor(self, extension): raise TypeError('Expected an extension descriptor.') if extension.extension_scope is None: + self._CheckConflictRegister( + extension, extension.full_name, extension.file.name) self._toplevel_extensions[extension.full_name] = extension try: @@ -372,12 +371,6 @@ def _InternalAddFileDescriptor(self, file_desc): """ self._AddFileDescriptor(file_desc) - # TODO(jieluo): This is a temporary solution for FieldDescriptor.file. - # FieldDescriptor.file is added in code gen. Remove this solution after - # maybe 2020 for compatibility reason (with 3.4.1 only). - for extension in file_desc.extensions_by_name.values(): - self._file_desc_by_toplevel_extension[ - extension.full_name] = file_desc def _AddFileDescriptor(self, file_desc): """Adds a FileDescriptor to the pool, non-recursively. @@ -483,7 +476,7 @@ def _InternalFindFileContainingSymbol(self, symbol): pass try: - return self._file_desc_by_toplevel_extension[symbol] + return self._toplevel_extensions[symbol].file except KeyError: pass @@ -792,8 +785,6 @@ def _ConvertFileProtoToFileDescriptor(self, file_proto): file_descriptor.package, scope) file_descriptor.extensions_by_name[extension_desc.name] = ( extension_desc) - self._file_desc_by_toplevel_extension[extension_desc.full_name] = ( - file_descriptor) for desc_proto in file_proto.message_type: self._SetAllFieldTypes(file_proto.package, desc_proto, scope) diff --git a/python/google/protobuf/internal/descriptor_pool_test.py b/python/google/protobuf/internal/descriptor_pool_test.py index 9e451b45beddc..efd909a1beb98 100644 --- a/python/google/protobuf/internal/descriptor_pool_test.py +++ b/python/google/protobuf/internal/descriptor_pool_test.py @@ -33,7 +33,6 @@ __author__ = 'matthewtoia@google.com (Matt Toia)' import copy -import os import unittest import warnings @@ -415,7 +414,6 @@ def testAddSerializedFile(self): field = file_json.message_types_by_name['class'].fields_by_name['int_field'] self.assertEqual(field.json_name, 'json_int') - def testEnumDefaultValue(self): """Test the default value of enums which don't start at zero.""" def _CheckDefaultValue(file_descriptor): diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index 18b784e5bf387..49361ee5eedc5 100644 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py @@ -1598,6 +1598,47 @@ def testParseAllowedUnknownExtension(self): self.assertEqual(23, message.message_set.Extensions[ext1].i) self.assertEqual('foo', message.message_set.Extensions[ext2].str) + # Handle Any messages inside unknown extensions. + message = any_test_pb2.TestAny() + text = ('any_value {\n' + ' [type.googleapis.com/google.protobuf.internal.TestAny] {\n' + ' [unknown_extension] {\n' + ' str: "string"\n' + ' any_value {\n' + ' [type.googleapis.com/protobuf_unittest.OneString] {\n' + ' data: "string"\n' + ' }\n' + ' }\n' + ' }\n' + ' }\n' + '}\n' + 'int32_value: 123') + text_format.Parse(text, message, allow_unknown_extension=True) + self.assertEqual(123, message.int32_value) + + # Fail if invalid Any message type url inside unknown extensions. + message = any_test_pb2.TestAny() + text = ('any_value {\n' + ' [type.googleapis.com.invalid/google.protobuf.internal.TestAny] {\n' + ' [unknown_extension] {\n' + ' str: "string"\n' + ' any_value {\n' + ' [type.googleapis.com/protobuf_unittest.OneString] {\n' + ' data: "string"\n' + ' }\n' + ' }\n' + ' }\n' + ' }\n' + '}\n' + 'int32_value: 123') + self.assertRaisesRegex( + text_format.ParseError, + '[type.googleapis.com.invalid/google.protobuf.internal.TestAny]', + text_format.Parse, + text, + message, + allow_unknown_extension=True) + def testParseBadIdentifier(self): message = unittest_pb2.TestAllTypes() text = ('optional_nested_message { "bb": 1 }') diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index 162531226e5f8..b434934ee138f 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -40,7 +40,6 @@ #include #include -#include #include #include #include @@ -48,6 +47,7 @@ #include #include #include +#include #include #define PyString_AsStringAndSize(ob, charpp, sizep) \ diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index 2c4a9573e79b2..abfe0454e95b5 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -51,8 +51,6 @@ #endif #include #include -#include -#include #include #include #include @@ -71,7 +69,9 @@ #include #include #include +#include #include +#include #include // clang-format off diff --git a/python/google/protobuf/text_format.py b/python/google/protobuf/text_format.py index a6d8bcf6482a8..fae5f23fe3208 100644 --- a/python/google/protobuf/text_format.py +++ b/python/google/protobuf/text_format.py @@ -856,9 +856,12 @@ def _ParseOrMerge(self, lines, message): ParseError: On text parsing problems. """ # Tokenize expects native str lines. - str_lines = ( - line if isinstance(line, str) else line.decode('utf-8') - for line in lines) + try: + str_lines = ( + line if isinstance(line, str) else line.decode('utf-8') + for line in lines) + except UnicodeDecodeError as e: + raise self._StringParseError(e) tokenizer = Tokenizer(str_lines) while not tokenizer.AtEnd(): self._MergeField(tokenizer, message) @@ -1190,10 +1193,17 @@ def _SkipField(tokenizer): tokenizer: A tokenizer to parse the field name and values. """ if tokenizer.TryConsume('['): - # Consume extension name. + # Consume extension or google.protobuf.Any type URL tokenizer.ConsumeIdentifier() + num_identifiers = 1 while tokenizer.TryConsume('.'): tokenizer.ConsumeIdentifier() + num_identifiers += 1 + # This is possibly a type URL for an Any message. + if num_identifiers == 3 and tokenizer.TryConsume('/'): + tokenizer.ConsumeIdentifier() + while tokenizer.TryConsume('.'): + tokenizer.ConsumeIdentifier() tokenizer.Consume(']') else: tokenizer.ConsumeIdentifierOrNumber() diff --git a/src/google/protobuf/arenaz_sampler.cc b/src/google/protobuf/arenaz_sampler.cc index 0eac693d98a1c..7383e3eb3b4a2 100644 --- a/src/google/protobuf/arenaz_sampler.cc +++ b/src/google/protobuf/arenaz_sampler.cc @@ -97,15 +97,12 @@ void RecordResetSlow(ThreadSafeArenaStats* info) { void RecordAllocateSlow(ThreadSafeArenaStats* info, size_t requested, size_t allocated, size_t wasted) { + info->num_allocations.fetch_add(1, std::memory_order_relaxed); info->bytes_requested.fetch_add(requested, std::memory_order_relaxed); info->bytes_allocated.fetch_add(allocated, std::memory_order_relaxed); info->bytes_wasted.fetch_add(wasted, std::memory_order_relaxed); - info->num_allocations.fetch_add(1, std::memory_order_relaxed); - const uint64_t tid = (1ULL << (GetCachedTID() % 63)); - const uint64_t thread_ids = info->thread_ids.load(std::memory_order_relaxed); - if (!(thread_ids & tid)) { - info->thread_ids.store(thread_ids | tid, std::memory_order_relaxed); - } + const uint64_t tid = 1ULL << (GetCachedTID() % 63); + info->thread_ids.fetch_or(tid, std::memory_order_relaxed); } ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) { diff --git a/src/google/protobuf/arenaz_sampler.h b/src/google/protobuf/arenaz_sampler.h index b04b0cc678c5b..76698a26a3ec7 100644 --- a/src/google/protobuf/arenaz_sampler.h +++ b/src/google/protobuf/arenaz_sampler.h @@ -70,8 +70,10 @@ struct ThreadSafeArenaStats std::atomic bytes_wasted; // Records the largest size an arena ever had. Maintained across resets. std::atomic max_bytes_allocated; - // Bit i when set to 1 indicates that a thread with tid % 63 = i accessed the - // underlying arena. The field is maintained across resets. + // Bit `i` is set to 1 indicates that a thread with `tid % 63 = i` accessed + // the underlying arena. We use `% 63` as a rudimentary hash to ensure some + // bit mixing for thread-ids; `% 64` would only grab the low bits and might + // create sampling artifacts. Maintained across resets. std::atomic thread_ids; // All of the fields below are set by `PrepareForSampling`, they must not diff --git a/src/google/protobuf/compiler/command_line_interface_unittest.cc b/src/google/protobuf/compiler/command_line_interface_unittest.cc index e88489753d3f3..99fa0ace39ddd 100644 --- a/src/google/protobuf/compiler/command_line_interface_unittest.cc +++ b/src/google/protobuf/compiler/command_line_interface_unittest.cc @@ -2697,7 +2697,7 @@ class EncodeDecodeTest : public testing::TestWithParam { TEST_P(EncodeDecodeTest, Encode) { RedirectStdinFromFile(TestUtil::GetTestDataPath( - "net/proto2/internal/" + "third_party/protobuf/" "testdata/text_format_unittest_data_oneof_implemented.txt")); std::string args; if (GetParam() != DESCRIPTOR_SET_IN) { @@ -2706,17 +2706,17 @@ TEST_P(EncodeDecodeTest, Encode) { } EXPECT_TRUE(Run(args + " --encode=protobuf_unittest.TestAllTypes")); ExpectStdoutMatchesBinaryFile(TestUtil::GetTestDataPath( - "net/proto2/internal/testdata/golden_message_oneof_implemented")); + "third_party/protobuf/testdata/golden_message_oneof_implemented")); } TEST_P(EncodeDecodeTest, Decode) { RedirectStdinFromFile(TestUtil::GetTestDataPath( - "net/proto2/internal/testdata/golden_message_oneof_implemented")); + "third_party/protobuf/testdata/golden_message_oneof_implemented")); EXPECT_TRUE( Run(TestUtil::MaybeTranslatePath("net/proto2/internal/unittest.proto") + " --decode=protobuf_unittest.TestAllTypes")); ExpectStdoutMatchesTextFile(TestUtil::GetTestDataPath( - "net/proto2/internal/" + "third_party/protobuf/" "testdata/text_format_unittest_data_oneof_implemented.txt")); } @@ -2764,7 +2764,7 @@ TEST_P(EncodeDecodeTest, ProtoParseError) { TEST_P(EncodeDecodeTest, EncodeDeterministicOutput) { RedirectStdinFromFile(TestUtil::GetTestDataPath( - "net/proto2/internal/" + "third_party/protobuf/" "testdata/text_format_unittest_data_oneof_implemented.txt")); std::string args; if (GetParam() != DESCRIPTOR_SET_IN) { @@ -2774,12 +2774,12 @@ TEST_P(EncodeDecodeTest, EncodeDeterministicOutput) { EXPECT_TRUE(Run( args + " --encode=protobuf_unittest.TestAllTypes --deterministic_output")); ExpectStdoutMatchesBinaryFile(TestUtil::GetTestDataPath( - "net/proto2/internal/testdata/golden_message_oneof_implemented")); + "third_party/protobuf/testdata/golden_message_oneof_implemented")); } TEST_P(EncodeDecodeTest, DecodeDeterministicOutput) { RedirectStdinFromFile(TestUtil::GetTestDataPath( - "net/proto2/internal/testdata/golden_message_oneof_implemented")); + "third_party/protobuf/testdata/golden_message_oneof_implemented")); EXPECT_FALSE( Run(TestUtil::MaybeTranslatePath("net/proto2/internal/unittest.proto") + " --decode=protobuf_unittest.TestAllTypes --deterministic_output")); diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 69069dacdd767..d3190fabc36d2 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -2101,7 +2101,9 @@ void MessageGenerator::GenerateClassMethods(io::Printer* printer) { if (!has_bit_indices_.empty()) { format( "using HasBits = " - "decltype(std::declval<$classname$>().$has_bits$);\n"); + "decltype(std::declval<$classname$>().$has_bits$);\n" + "static constexpr int32_t kHasBitsOffset =\n" + " 8 * PROTOBUF_FIELD_OFFSET($classname$, _impl_._has_bits_);\n"); } for (auto field : FieldRange(descriptor_)) { field_generators_.get(field).GenerateInternalAccessorDeclarations(printer); diff --git a/src/google/protobuf/compiler/cpp/parse_function_generator.cc b/src/google/protobuf/compiler/cpp/parse_function_generator.cc index 0f1d767c816b9..91ceeeb257b8b 100644 --- a/src/google/protobuf/compiler/cpp/parse_function_generator.cc +++ b/src/google/protobuf/compiler/cpp/parse_function_generator.cc @@ -47,6 +47,15 @@ namespace { using google::protobuf::internal::WireFormat; using google::protobuf::internal::WireFormatLite; +bool UseDirectTcParserTable(const FieldDescriptor* field, + const Options& options) { + auto* m = field->message_type(); + return !m->options().message_set_wire_format() && + m->file()->options().optimize_for() != FileOptions::CODE_SIZE && + !HasSimpleBaseClass(m, options) && !HasTracker(m, options) + ; +} + std::vector GetOrderedFields( const Descriptor* descriptor, const Options& options) { std::vector ordered_fields; @@ -324,10 +333,17 @@ TailCallTableInfo::TailCallTableInfo( // Lazy fields are handled by the generated fallback function. } else { field_entries.back().aux_idx = aux_entries.size(); - const Descriptor* field_type = field->message_type(); - aux_entries.push_back(StrCat( - "reinterpret_cast(&", QualifiedDefaultInstanceName(field_type, options), ")")); + if (UseDirectTcParserTable(field, options)) { + const Descriptor* field_type = field->message_type(); + aux_entries.push_back( + StrCat("::_pbi::TcParser::GetTable<", + QualifiedClassName(field_type, options), ">()")); + } else { + const Descriptor* field_type = field->message_type(); + aux_entries.push_back( + StrCat("::_pbi::FieldAuxDefaultMessage{}, &", + QualifiedDefaultInstanceName(field_type, options))); + } } } else if (field->type() == FieldDescriptor::TYPE_ENUM && !HasPreservingUnknownEnumSemantics(field)) { @@ -518,6 +534,9 @@ bool ParseFunctionGenerator::should_generate_tctable() const { if (options_.tctable_mode == Options::kTCTableNever) { return false; } + if (HasSimpleBaseClass(descriptor_, options_)) { + return false; + } return true; } @@ -547,7 +566,7 @@ void ParseFunctionGenerator::GenerateTailcallFallbackFunction( if (num_hasbits_ > 0) { // Sync hasbits - format("typed_msg->_impl_._has_bits_[0] = hasbits;\n"); + format("typed_msg->_impl_._has_bits_[0] |= hasbits;\n"); } format("uint32_t tag = data.tag();\n"); @@ -604,6 +623,7 @@ void ParseFunctionGenerator::GenerateDataDecls(io::Printer* printer) { } auto field_num_to_entry_table = MakeNumToEntryTable(ordered_fields_); format( + "friend class ::$proto_ns$::internal::TcParser;\n" "static const ::$proto_ns$::internal::" "TcParseTable<$1$, $2$, $3$, $4$, $5$> _table_;\n", tc_table_info_->table_size_log2, ordered_fields_.size(), @@ -761,7 +781,7 @@ void ParseFunctionGenerator::GenerateTailCallTable(Formatter& format) { // unknown fields and potentially an extension range. auto field_num_to_entry_table = MakeNumToEntryTable(ordered_fields_); format( - "PROTOBUF_ATTRIBUTE_INIT_PRIORITY1\n" + "PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1\n" "const ::_pbi::TcParseTable<$1$, $2$, $3$, $4$, $5$> " "$classname$::_table_ = " "{\n", @@ -1012,6 +1032,11 @@ static void FormatFieldKind(Formatter& format, case FieldDescriptor::TYPE_GROUP: format("Message | ::_fl::kRepGroup"); + if (UseDirectTcParserTable(field, options)) { + format(" | ::_fl::kTvTable"); + } else { + format(" | ::_fl::kTvDefault"); + } break; case FieldDescriptor::TYPE_MESSAGE: if (field->is_map()) { @@ -1023,6 +1048,11 @@ static void FormatFieldKind(Formatter& format, } else if (IsImplicitWeakField(field, options, scc_analyzer)) { format(" | ::_fl::kRepIWeak"); } + if (UseDirectTcParserTable(field, options)) { + format(" | ::_fl::kTvTable"); + } else { + format(" | ::_fl::kTvDefault"); + } } break; } @@ -1052,11 +1082,22 @@ void ParseFunctionGenerator::GenerateFieldEntries(Formatter& format) { } else { const OneofDescriptor* oneof = field->real_containing_oneof(); bool cold = ShouldSplit(field, options_); - format("PROTOBUF_FIELD_OFFSET($classname$$1$, $2$), $3$, $4$,\n ", + format("PROTOBUF_FIELD_OFFSET($classname$$1$, $2$), ", cold ? "::Impl_::Split" : "", cold ? FieldName(field) + "_" - : FieldMemberName(field, /*cold=*/false), - (oneof ? oneof->index() : entry.hasbit_idx), entry.aux_idx); + : FieldMemberName(field, /*cold=*/false)); + if (oneof) { + format("$1$, ", oneof->index()); + } else if (num_hasbits_ > 0 || IsMapEntryMessage(descriptor_)) { + if (entry.hasbit_idx >= 0) { + format("_Internal::kHasBitsOffset + $1$, ", entry.hasbit_idx); + } else { + format("$1$, ", entry.hasbit_idx); + } + } else { + format("0, "); + } + format("$1$,\n ", entry.aux_idx); FormatFieldKind(format, entry, options_, scc_analyzer_); } format("},\n"); @@ -1691,10 +1732,10 @@ std::string FieldParseFunctionName( break; case FieldDescriptor::TYPE_MESSAGE: - name.append("M"); + name.append(UseDirectTcParserTable(field, options) ? "Mt" : "Md"); break; case FieldDescriptor::TYPE_GROUP: - name.append("G"); + name.append(UseDirectTcParserTable(field, options) ? "Gt" : "Gd"); break; default: diff --git a/src/google/protobuf/compiler/cpp/test_bad_identifiers.proto b/src/google/protobuf/compiler/cpp/test_bad_identifiers.proto index 466a84194af07..028248e611c30 100644 --- a/src/google/protobuf/compiler/cpp/test_bad_identifiers.proto +++ b/src/google/protobuf/compiler/cpp/test_bad_identifiers.proto @@ -61,7 +61,9 @@ message TestConflictingSymbolNames { optional int32 total_size = 6; optional int32 tag = 7; - enum TestEnum { FOO = 0; } + enum TestEnum { + FOO = 0; + } message Data1 { repeated int32 data = 1; } diff --git a/src/google/protobuf/compiler/cpp/unittest.inc b/src/google/protobuf/compiler/cpp/unittest.inc index 0b4717611c79d..0f841a458b7a5 100644 --- a/src/google/protobuf/compiler/cpp/unittest.inc +++ b/src/google/protobuf/compiler/cpp/unittest.inc @@ -56,26 +56,25 @@ // visual studio to compile (report internal errors). #include #endif +#include +#include +#include #include -#include #include #include +#include +#include +#include +#include #include +#include +#include +#include #include #include -#include -#include #include #include #include - -#include -#include -#include -#include -#include -#include -#include #include // Must be included last. diff --git a/src/google/protobuf/compiler/java/enum_field.cc b/src/google/protobuf/compiler/java/enum_field.cc index ed3594ce70fb1..4a93addbe1d4f 100644 --- a/src/google/protobuf/compiler/java/enum_field.cc +++ b/src/google/protobuf/compiler/java/enum_field.cc @@ -42,7 +42,6 @@ #include #include #include -#include #include #include #include @@ -85,13 +84,6 @@ void SetEnumVariables(const FieldDescriptor* descriptor, int messageBitIndex, ? "@kotlin.Deprecated(message = \"Field " + (*variables)["name"] + " is deprecated\") " : ""; - (*variables)["on_changed"] = "onChanged();"; - // Use deprecated valueOf() method to be compatible with old generated code - // for v2.5.0/v2.6.1. - // TODO(xiaofeng): Use "forNumber" when we no longer support compatibility - // with v2.5.0/v2.6.1, and remove the @SuppressWarnings annotations. - (*variables)["for_number"] = "valueOf"; - if (HasHasbit(descriptor)) { // For singular messages and builders, one bit is used for the hasField bit. (*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex); @@ -204,8 +196,7 @@ void ImmutableEnumFieldGenerator::GenerateMembers(io::Printer* printer) const { printer->Print(variables_, "@java.lang.Override $deprecation$public $type$ " "${$get$capitalized_name$$}$() {\n" - " @SuppressWarnings(\"deprecation\")\n" - " $type$ result = $type$.$for_number$($name$_);\n" + " $type$ result = $type$.forNumber($name$_);\n" " return result == null ? $unknown$ : result;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -238,7 +229,7 @@ void ImmutableEnumFieldGenerator::GenerateBuilderMembers( "${$set$capitalized_name$Value$}$(int value) {\n" " $set_has_field_bit_builder$\n" " $name$_ = value;\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -247,8 +238,7 @@ void ImmutableEnumFieldGenerator::GenerateBuilderMembers( printer->Print(variables_, "@java.lang.Override\n" "$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n" - " @SuppressWarnings(\"deprecation\")\n" - " $type$ result = $type$.$for_number$($name$_);\n" + " $type$ result = $type$.forNumber($name$_);\n" " return result == null ? $unknown$ : result;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -262,7 +252,7 @@ void ImmutableEnumFieldGenerator::GenerateBuilderMembers( " }\n" " $set_has_field_bit_builder$\n" " $name$_ = value.getNumber();\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -273,7 +263,7 @@ void ImmutableEnumFieldGenerator::GenerateBuilderMembers( "$deprecation$public Builder ${$clear$capitalized_name$$}$() {\n" " $clear_has_field_bit_builder$\n" " $name$_ = $default_number$;\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -364,8 +354,7 @@ void ImmutableEnumFieldGenerator::GenerateParsingCode( } else { printer->Print(variables_, "int rawValue = input.readEnum();\n" - " @SuppressWarnings(\"deprecation\")\n" - "$type$ value = $type$.$for_number$(rawValue);\n" + "$type$ value = $type$.forNumber(rawValue);\n" "if (value == null) {\n" " unknownFields.mergeVarintField($number$, rawValue);\n" "} else {\n" @@ -453,8 +442,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateMembers( printer->Print(variables_, "$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n" " if ($has_oneof_case_message$) {\n" - " @SuppressWarnings(\"deprecation\")\n" - " $type$ result = $type$.$for_number$(\n" + " $type$ result = $type$.forNumber(\n" " (java.lang.Integer) $oneof_name$_);\n" " return result == null ? $unknown$ : result;\n" " }\n" @@ -493,7 +481,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateBuilderMembers( "${$set$capitalized_name$Value$}$(int value) {\n" " $set_oneof_case_message$;\n" " $oneof_name$_ = value;\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -503,8 +491,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateBuilderMembers( "@java.lang.Override\n" "$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n" " if ($has_oneof_case_message$) {\n" - " @SuppressWarnings(\"deprecation\")\n" - " $type$ result = $type$.$for_number$(\n" + " $type$ result = $type$.forNumber(\n" " (java.lang.Integer) $oneof_name$_);\n" " return result == null ? $unknown$ : result;\n" " }\n" @@ -521,7 +508,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateBuilderMembers( " }\n" " $set_oneof_case_message$;\n" " $oneof_name$_ = value.getNumber();\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -533,7 +520,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateBuilderMembers( " if ($has_oneof_case_message$) {\n" " $clear_oneof_case_message$;\n" " $oneof_name$_ = null;\n" - " $on_changed$\n" + " onChanged();\n" " }\n" " return this;\n" "}\n"); @@ -570,8 +557,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateParsingCode( } else { printer->Print(variables_, "int rawValue = input.readEnum();\n" - "@SuppressWarnings(\"deprecation\")\n" - "$type$ value = $type$.$for_number$(rawValue);\n" + "$type$ value = $type$.forNumber(rawValue);\n" "if (value == null) {\n" " unknownFields.mergeVarintField($number$, rawValue);\n" "} else {\n" @@ -685,8 +671,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateMembers( " new com.google.protobuf.Internal.ListAdapter.Converter<\n" " java.lang.Integer, $type$>() {\n" " public $type$ convert(java.lang.Integer from) {\n" - " @SuppressWarnings(\"deprecation\")\n" - " $type$ result = $type$.$for_number$(from);\n" + " $type$ result = $type$.forNumber(from);\n" " return result == null ? $unknown$ : result;\n" " }\n" " };\n"); @@ -802,7 +787,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( " }\n" " ensure$capitalized_name$IsMutable();\n" " $name$_.set(index, value.getNumber());\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -816,7 +801,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( " }\n" " ensure$capitalized_name$IsMutable();\n" " $name$_.add(value.getNumber());\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -829,7 +814,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( " for ($type$ value : values) {\n" " $name$_.add(value.getNumber());\n" " }\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -840,7 +825,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( "$deprecation$public Builder ${$clear$capitalized_name$$}$() {\n" " $name$_ = java.util.Collections.emptyList();\n" " $clear_mutable_bit_builder$;\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -870,7 +855,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( " int index, int value) {\n" " ensure$capitalized_name$IsMutable();\n" " $name$_.set(index, value);\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -881,7 +866,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( "${$add$capitalized_name$Value$}$(int value) {\n" " ensure$capitalized_name$IsMutable();\n" " $name$_.add(value);\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -895,7 +880,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( " for (int value : values) {\n" " $name$_.add(value);\n" " }\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -935,7 +920,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateMergingCode( " ensure$capitalized_name$IsMutable();\n" " $name$_.addAll(other.$name$_);\n" " }\n" - " $on_changed$\n" + " onChanged();\n" "}\n"); } @@ -967,8 +952,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateParsingCode( printer->Print( variables_, "int rawValue = input.readEnum();\n" - "@SuppressWarnings(\"deprecation\")\n" - "$type$ value = $type$.$for_number$(rawValue);\n" + "$type$ value = $type$.forNumber(rawValue);\n" "if (value == null) {\n" " unknownFields.mergeVarintField($number$, rawValue);\n" "} else {\n" diff --git a/src/google/protobuf/compiler/java/map_field.cc b/src/google/protobuf/compiler/java/map_field.cc index bc48f61a7f383..ce0766129c3bc 100644 --- a/src/google/protobuf/compiler/java/map_field.cc +++ b/src/google/protobuf/compiler/java/map_field.cc @@ -127,6 +127,8 @@ void SetMessageVariables(const FieldDescriptor* descriptor, int messageBitIndex, if (valueJavaType == JAVATYPE_ENUM) { // We store enums as Integers internally. (*variables)["value_type"] = "int"; + (*variables)["value_type_pass_through_nullness"] = + (*variables)["value_type"]; (*variables)["boxed_value_type"] = "java.lang.Integer"; (*variables)["value_wire_type"] = WireType(value); (*variables)["value_default_value"] = @@ -262,9 +264,10 @@ void ImmutableMapFieldGenerator::GenerateInterfaceMembers( WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, "$deprecation$\n" - "$value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" + "$value_type_pass_through_nullness$ " + "${$get$capitalized_name$ValueOrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue);\n"); + " $value_type_pass_through_nullness$ defaultValue);\n"); printer->Annotate("{", "}", descriptor_); WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, @@ -609,9 +612,10 @@ void ImmutableMapFieldGenerator::GenerateMapGetters( variables_, "@java.lang.Override\n" "$deprecation$\n" - "public $value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" + "public $value_type_pass_through_nullness$ " + "${$get$capitalized_name$ValueOrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue) {\n" + " $value_type_pass_through_nullness$ defaultValue) {\n" " $key_null_check$\n" " java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n" " internalGet$capitalized_name$().getMap();\n" @@ -661,9 +665,10 @@ void ImmutableMapFieldGenerator::GenerateMapGetters( variables_, "@java.lang.Override\n" "$deprecation$\n" - "public $value_type$ ${$get$capitalized_name$OrDefault$}$(\n" + "public $value_type_pass_through_nullness$ " + "${$get$capitalized_name$OrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue) {\n" + " $value_type_pass_through_nullness$ defaultValue) {\n" " $key_null_check$\n" " java.util.Map<$type_parameters$> map =\n" " internalGet$capitalized_name$().getMap();\n" diff --git a/src/google/protobuf/compiler/java/map_field_lite.cc b/src/google/protobuf/compiler/java/map_field_lite.cc index 4f2766e8ff5a6..7e5f829ba47d2 100644 --- a/src/google/protobuf/compiler/java/map_field_lite.cc +++ b/src/google/protobuf/compiler/java/map_field_lite.cc @@ -126,6 +126,8 @@ void SetMessageVariables(const FieldDescriptor* descriptor, int messageBitIndex, if (GetJavaType(value) == JAVATYPE_ENUM) { // We store enums as Integers internally. (*variables)["value_type"] = "int"; + (*variables)["value_type_pass_through_nullness"] = + (*variables)["value_type"]; (*variables)["boxed_value_type"] = "java.lang.Integer"; (*variables)["value_wire_type"] = WireType(value); (*variables)["value_default_value"] = @@ -247,9 +249,10 @@ void ImmutableMapFieldLiteGenerator::GenerateInterfaceMembers( WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, "$deprecation$\n" - "$value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" + "$value_type_pass_through_nullness$ " + "${$get$capitalized_name$ValueOrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue);\n"); + " $value_type_pass_through_nullness$ defaultValue);\n"); printer->Annotate("{", "}", descriptor_); WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, @@ -373,9 +376,10 @@ void ImmutableMapFieldLiteGenerator::GenerateMembers( variables_, "@java.lang.Override\n" "$deprecation$\n" - "public $value_enum_type$ ${$get$capitalized_name$OrDefault$}$(\n" + "public $value_enum_type_pass_through_nullness$ " + "${$get$capitalized_name$OrDefault$}$(\n" " $key_type$ key,\n" - " $value_enum_type$ defaultValue) {\n" + " $value_enum_type_pass_through_nullness$ defaultValue) {\n" " $key_null_check$\n" " java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n" " internalGet$capitalized_name$();\n" @@ -429,9 +433,10 @@ void ImmutableMapFieldLiteGenerator::GenerateMembers( variables_, "@java.lang.Override\n" "$deprecation$\n" - "public $value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" + "public $value_type_pass_through_nullness$ " + "${$get$capitalized_name$ValueOrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue) {\n" + " $value_type_pass_through_nullness$ defaultValue) {\n" " $key_null_check$\n" " java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n" " internalGet$capitalized_name$();\n" @@ -482,9 +487,10 @@ void ImmutableMapFieldLiteGenerator::GenerateMembers( variables_, "@java.lang.Override\n" "$deprecation$\n" - "public $value_type$ ${$get$capitalized_name$OrDefault$}$(\n" + "public $value_type_pass_through_nullness$ " + "${$get$capitalized_name$OrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue) {\n" + " $value_type_pass_through_nullness$ defaultValue) {\n" " $key_null_check$\n" " java.util.Map<$type_parameters$> map =\n" " internalGet$capitalized_name$();\n" @@ -701,9 +707,10 @@ void ImmutableMapFieldLiteGenerator::GenerateBuilderMembers( variables_, "@java.lang.Override\n" "$deprecation$\n" - "public $value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" + "public $value_type_pass_through_nullness$ " + "${$get$capitalized_name$ValueOrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue) {\n" + " $value_type_pass_through_nullness$ defaultValue) {\n" " $key_null_check$\n" " java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n" " instance.get$capitalized_name$ValueMap();\n" @@ -776,9 +783,10 @@ void ImmutableMapFieldLiteGenerator::GenerateBuilderMembers( variables_, "@java.lang.Override\n" "$deprecation$\n" - "public $value_type$ ${$get$capitalized_name$OrDefault$}$(\n" + "public $value_type_pass_through_nullness$ " + "${$get$capitalized_name$OrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue) {\n" + " $value_type_pass_through_nullness$ defaultValue) {\n" " $key_null_check$\n" " java.util.Map<$type_parameters$> map =\n" " instance.get$capitalized_name$Map();\n" diff --git a/src/google/protobuf/compiler/java/message.cc b/src/google/protobuf/compiler/java/message.cc index 0cc6171731997..95917a7f6dac4 100644 --- a/src/google/protobuf/compiler/java/message.cc +++ b/src/google/protobuf/compiler/java/message.cc @@ -425,6 +425,8 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) { vars["oneof_capitalized_name"] = context_->GetOneofGeneratorInfo(oneof)->capitalized_name; vars["oneof_index"] = StrCat((oneof)->index()); + vars["{"] = ""; + vars["}"] = ""; // oneofCase_ and oneof_ printer->Print(vars, "private int $oneof_name$Case_ = 0;\n" @@ -432,11 +434,12 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) { // OneofCase enum printer->Print( vars, - "public enum $oneof_capitalized_name$Case\n" + "public enum ${$$oneof_capitalized_name$Case$}$\n" // TODO(dweis): Remove EnumLite when we want to break compatibility with // 3.x users " implements com.google.protobuf.Internal.EnumLite,\n" " com.google.protobuf.AbstractMessage.InternalOneOfEnum {\n"); + printer->Annotate("{", "}", oneof); printer->Indent(); for (int j = 0; j < (oneof)->field_count(); j++) { const FieldDescriptor* field = (oneof)->field(j); @@ -445,6 +448,7 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) { field->options().deprecated() ? "@java.lang.Deprecated " : "", "field_name", ToUpper(field->name()), "field_number", StrCat(field->number())); + printer->Annotate("field_name", field); } printer->Print("$cap_oneof_name$_NOT_SET(0);\n", "cap_oneof_name", ToUpper(vars["oneof_name"])); @@ -487,11 +491,12 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) { // oneofCase() printer->Print(vars, "public $oneof_capitalized_name$Case\n" - "get$oneof_capitalized_name$Case() {\n" + "${$get$oneof_capitalized_name$Case$}$() {\n" " return $oneof_capitalized_name$Case.forNumber(\n" " $oneof_name$Case_);\n" "}\n" "\n"); + printer->Annotate("{", "}", oneof); } if (IsAnyMessage(descriptor_)) { @@ -1012,6 +1017,8 @@ void ImmutableMessageGenerator::GenerateEqualsAndHashCode( " return true;\n" "}\n" "if (!(obj instanceof $classname$)) {\n" + // don't simply return false because mutable and immutable types + // can be equal " return super.equals(obj);\n" "}\n" "$classname$ other = ($classname$) obj;\n" diff --git a/src/google/protobuf/compiler/plugin.pb.cc b/src/google/protobuf/compiler/plugin.pb.cc index 8f8d83f02e469..def353e2153ab 100644 --- a/src/google/protobuf/compiler/plugin.pb.cc +++ b/src/google/protobuf/compiler/plugin.pb.cc @@ -234,6 +234,8 @@ constexpr int CodeGeneratorResponse::Feature_ARRAYSIZE; class Version::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(Version, _impl_._has_bits_); static void set_has_major(HasBits* has_bits) { (*has_bits)[0] |= 2u; } @@ -558,6 +560,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata Version::GetMetadata() const { class CodeGeneratorRequest::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(CodeGeneratorRequest, _impl_._has_bits_); static void set_has_parameter(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -910,6 +914,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata CodeGeneratorRequest::GetMetadata() const { class CodeGeneratorResponse_File::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(CodeGeneratorResponse_File, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -1300,6 +1306,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata CodeGeneratorResponse_File::GetMetadata() cons class CodeGeneratorResponse::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(CodeGeneratorResponse, _impl_._has_bits_); static void set_has_error(HasBits* has_bits) { (*has_bits)[0] |= 1u; } diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index d3bfb461eadaa..645096686d3f0 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -1512,6 +1512,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata FileDescriptorSet::GetMetadata() const { class FileDescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(FileDescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -2209,6 +2211,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata FileDescriptorProto::GetMetadata() const { class DescriptorProto_ExtensionRange::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(DescriptorProto_ExtensionRange, _impl_._has_bits_); static void set_has_start(HasBits* has_bits) { (*has_bits)[0] |= 2u; } @@ -2493,6 +2497,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata DescriptorProto_ExtensionRange::GetMetadata() class DescriptorProto_ReservedRange::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(DescriptorProto_ReservedRange, _impl_._has_bits_); static void set_has_start(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -2730,6 +2736,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata DescriptorProto_ReservedRange::GetMetadata() c class DescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(DescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -3505,6 +3513,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata ExtensionRangeOptions::GetMetadata() const { class FieldDescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(FieldDescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -4186,6 +4196,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata FieldDescriptorProto::GetMetadata() const { class OneofDescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(OneofDescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -4461,6 +4473,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata OneofDescriptorProto::GetMetadata() const { class EnumDescriptorProto_EnumReservedRange::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(EnumDescriptorProto_EnumReservedRange, _impl_._has_bits_); static void set_has_start(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -4698,6 +4712,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata EnumDescriptorProto_EnumReservedRange::GetMeta class EnumDescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(EnumDescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -5084,6 +5100,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata EnumDescriptorProto::GetMetadata() const { class EnumValueDescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(EnumValueDescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -5395,6 +5413,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata EnumValueDescriptorProto::GetMetadata() const class ServiceDescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(ServiceDescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -5706,6 +5726,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata ServiceDescriptorProto::GetMetadata() const { class MethodDescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(MethodDescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -6163,6 +6185,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata MethodDescriptorProto::GetMetadata() const { class FileOptions::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(FileOptions, _impl_._has_bits_); static void set_has_java_package(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -7282,6 +7306,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata FileOptions::GetMetadata() const { class MessageOptions::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(MessageOptions, _impl_._has_bits_); static void set_has_message_set_wire_format(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -7630,6 +7656,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata MessageOptions::GetMetadata() const { class FieldOptions::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(FieldOptions, _impl_._has_bits_); static void set_has_ctype(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -8286,6 +8314,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata OneofOptions::GetMetadata() const { class EnumOptions::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(EnumOptions, _impl_._has_bits_); static void set_has_allow_alias(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -8578,6 +8608,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata EnumOptions::GetMetadata() const { class EnumValueOptions::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(EnumValueOptions, _impl_._has_bits_); static void set_has_deprecated(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -8827,6 +8859,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata EnumValueOptions::GetMetadata() const { class ServiceOptions::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(ServiceOptions, _impl_._has_bits_); static void set_has_deprecated(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -9076,6 +9110,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata ServiceOptions::GetMetadata() const { class MethodOptions::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(MethodOptions, _impl_._has_bits_); static void set_has_deprecated(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -9377,6 +9413,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata MethodOptions::GetMetadata() const { class UninterpretedOption_NamePart::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(UninterpretedOption_NamePart, _impl_._has_bits_); static void set_has_name_part(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -9653,6 +9691,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata UninterpretedOption_NamePart::GetMetadata() co class UninterpretedOption::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(UninterpretedOption, _impl_._has_bits_); static void set_has_identifier_value(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -10122,6 +10162,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata UninterpretedOption::GetMetadata() const { class SourceCodeInfo_Location::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(SourceCodeInfo_Location, _impl_._has_bits_); static void set_has_leading_comments(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -10716,6 +10758,8 @@ ::PROTOBUF_NAMESPACE_ID::Metadata SourceCodeInfo::GetMetadata() const { class GeneratedCodeInfo_Annotation::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(GeneratedCodeInfo_Annotation, _impl_._has_bits_); static void set_has_source_file(HasBits* has_bits) { (*has_bits)[0] |= 1u; } diff --git a/src/google/protobuf/descriptor.proto b/src/google/protobuf/descriptor.proto index f8eb216cdcc36..5b4d06b41c093 100644 --- a/src/google/protobuf/descriptor.proto +++ b/src/google/protobuf/descriptor.proto @@ -604,11 +604,8 @@ message FieldOptions { // check its required fields, regardless of whether or not the message has // been parsed. // - // As of 2021, lazy does no correctness checks on the byte stream during - // parsing. This may lead to crashes if and when an invalid byte stream is - // finally parsed upon access. - // - // TODO(b/211906113): Enable validation on lazy fields. + // As of May 2022, lazy verifies the contents of the byte stream during + // parsing. An invalid byte stream will cause the overall parsing to fail. optional bool lazy = 5 [default = false]; // unverified_lazy does no correctness checks on the byte stream. This should diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 2a807e066bb74..53ca9acd326a5 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -983,8 +983,6 @@ void Reflection::Swap(Message* message1, Message* message2) const { return; } - GOOGLE_DCHECK_EQ(message1->GetOwningArena(), message2->GetOwningArena()); - UnsafeArenaSwap(message1, message2); } @@ -1014,9 +1012,6 @@ void Reflection::SwapFieldsImpl( std::set swapped_oneof; - GOOGLE_DCHECK(!unsafe_shallow_swap || message1->GetArenaForAllocation() == - message2->GetArenaForAllocation()); - const Message* prototype = message_factory_->GetPrototype(message1->GetDescriptor()); for (const auto* field : fields) { @@ -1073,6 +1068,9 @@ void Reflection::SwapFields( void Reflection::UnsafeShallowSwapFields( Message* message1, Message* message2, const std::vector& fields) const { + GOOGLE_DCHECK_EQ(message1->GetArenaForAllocation(), + message2->GetArenaForAllocation()); + SwapFieldsImpl(message1, message2, fields); } @@ -1103,6 +1101,11 @@ bool Reflection::HasField(const Message& message, } void Reflection::UnsafeArenaSwap(Message* lhs, Message* rhs) const { + GOOGLE_DCHECK_EQ(lhs->GetOwningArena(), rhs->GetOwningArena()); + InternalSwap(lhs, rhs); +} + +void Reflection::InternalSwap(Message* lhs, Message* rhs) const { if (lhs == rhs) return; MutableInternalMetadata(lhs)->InternalSwap(MutableInternalMetadata(rhs)); diff --git a/src/google/protobuf/generated_message_tctable_decl.h b/src/google/protobuf/generated_message_tctable_decl.h index b1bb1def7089e..3974dc34a7ee0 100644 --- a/src/google/protobuf/generated_message_tctable_decl.h +++ b/src/google/protobuf/generated_message_tctable_decl.h @@ -120,6 +120,8 @@ struct Offset { #pragma warning(disable : 4324) #endif +struct FieldAuxDefaultMessage {}; + // Base class for message-level table with info for the tail-call parser. struct alignas(uint64_t) TcParseTableBase { // Common attributes for message layout: @@ -191,7 +193,7 @@ struct alignas(uint64_t) TcParseTableBase { // Field entry for all fields. struct FieldEntry { uint32_t offset; // offset in the message object - int32_t has_idx; // has-bit index + int32_t has_idx; // has-bit index, relative to the message object uint16_t aux_idx; // index for `field_aux`. uint16_t type_card; // `FieldType` and `Cardinality` (see _impl.h) }; @@ -204,20 +206,28 @@ struct alignas(uint64_t) TcParseTableBase { // Auxiliary entries for field types that need extra information. union FieldAux { - constexpr FieldAux() : message_default(nullptr) {} + constexpr FieldAux() : message_default_p(nullptr) {} constexpr FieldAux(bool (*enum_validator)(int)) : enum_validator(enum_validator) {} constexpr FieldAux(field_layout::Offset off) : offset(off.off) {} constexpr FieldAux(int16_t range_start, uint16_t range_length) : enum_range{range_start, range_length} {} - constexpr FieldAux(const MessageLite* msg) : message_default(msg) {} + constexpr FieldAux(const MessageLite* msg) : message_default_p(msg) {} + constexpr FieldAux(FieldAuxDefaultMessage, const void* msg) + : message_default_p(msg) {} + constexpr FieldAux(const TcParseTableBase* table) : table(table) {} bool (*enum_validator)(int); struct { int16_t start; // minimum enum number (if it fits) uint16_t length; // length of range (i.e., max = start + length - 1) } enum_range; uint32_t offset; - const MessageLite* message_default; + const void* message_default_p; + const TcParseTableBase* table; + + const MessageLite* message_default() const { + return static_cast(message_default_p); + } }; const FieldAux* field_aux(uint32_t idx) const { return reinterpret_cast(reinterpret_cast(this) + diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index 21fa5332d39b2..c64b264dc7c38 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -147,6 +147,10 @@ enum TransformValidation : uint16_t { // String fields: kTvUtf8Debug = 1 << kTvShift, // proto2 kTvUtf8 = 2 << kTvShift, // proto3 + + // Message fields: + kTvDefault = 1 << kTvShift, // Aux has default_instance + kTvTable = 2 << kTvShift, // Aux has TcParseTableBase* }; static_assert((kTvEnum & kTvRange) != 0, @@ -262,6 +266,11 @@ extern template void AlignFail<8>(uintptr_t); // TcParser implements most of the parsing logic for tailcall tables. class PROTOBUF_EXPORT TcParser final { public: + template + static constexpr const TcParseTableBase* GetTable() { + return &T::_table_.header; + } + static const char* GenericFallback(PROTOBUF_TC_PARAM_DECL); static const char* GenericFallbackLite(PROTOBUF_TC_PARAM_DECL); @@ -364,16 +373,26 @@ class PROTOBUF_EXPORT TcParser final { // Functions referenced by generated fast tables (message types): // M: message G: group + // d: default* t: TcParseTable* (the contents of aux) // S: singular R: repeated // 1/2: tag length (bytes) - static const char* FastMS1(PROTOBUF_TC_PARAM_DECL); - static const char* FastMS2(PROTOBUF_TC_PARAM_DECL); - static const char* FastMR1(PROTOBUF_TC_PARAM_DECL); - static const char* FastMR2(PROTOBUF_TC_PARAM_DECL); - static const char* FastGS1(PROTOBUF_TC_PARAM_DECL); - static const char* FastGS2(PROTOBUF_TC_PARAM_DECL); - static const char* FastGR1(PROTOBUF_TC_PARAM_DECL); - static const char* FastGR2(PROTOBUF_TC_PARAM_DECL); + static const char* FastMdS1(PROTOBUF_TC_PARAM_DECL); + static const char* FastMdS2(PROTOBUF_TC_PARAM_DECL); + static const char* FastGdS1(PROTOBUF_TC_PARAM_DECL); + static const char* FastGdS2(PROTOBUF_TC_PARAM_DECL); + static const char* FastMtS1(PROTOBUF_TC_PARAM_DECL); + static const char* FastMtS2(PROTOBUF_TC_PARAM_DECL); + static const char* FastGtS1(PROTOBUF_TC_PARAM_DECL); + static const char* FastGtS2(PROTOBUF_TC_PARAM_DECL); + + static const char* FastMdR1(PROTOBUF_TC_PARAM_DECL); + static const char* FastMdR2(PROTOBUF_TC_PARAM_DECL); + static const char* FastGdR1(PROTOBUF_TC_PARAM_DECL); + static const char* FastGdR2(PROTOBUF_TC_PARAM_DECL); + static const char* FastMtR1(PROTOBUF_TC_PARAM_DECL); + static const char* FastMtR2(PROTOBUF_TC_PARAM_DECL); + static const char* FastGtR1(PROTOBUF_TC_PARAM_DECL); + static const char* FastGtR2(PROTOBUF_TC_PARAM_DECL); template static inline T& RefAt(void* x, size_t offset) { @@ -421,9 +440,9 @@ class PROTOBUF_EXPORT TcParser final { private: friend class GeneratedTcTableLiteTest; - template + template static inline const char* SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL); - template + template static inline const char* RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL); static inline PROTOBUF_ALWAYS_INLINE void SyncHasbits( @@ -432,7 +451,7 @@ class PROTOBUF_EXPORT TcParser final { if (has_bits_offset) { // Only the first 32 has-bits are updated. Nothing above those is stored, // but e.g. messages without has-bits update the upper bits. - RefAt(msg, has_bits_offset) = static_cast(hasbits); + RefAt(msg, has_bits_offset) |= static_cast(hasbits); } } diff --git a/src/google/protobuf/generated_message_tctable_lite.cc b/src/google/protobuf/generated_message_tctable_lite.cc index 9993811dca322..519b10776c4c6 100644 --- a/src/google/protobuf/generated_message_tctable_lite.cc +++ b/src/google/protobuf/generated_message_tctable_lite.cc @@ -85,10 +85,7 @@ PROTOBUF_NOINLINE const char* TcParser::ParseLoop( const TcParseTableBase* table) { ScopedArenaSwap saved(msg, ctx); while (!ctx->Done(&ptr)) { - // Unconditionally read has bits, even if we don't have has bits. - // has_bits_offset will be 0 and we will just read something valid. - uint64_t hasbits = ReadAt(msg, table->has_bits_offset); - ptr = TagDispatch(msg, ptr, ctx, table, hasbits, {}); + ptr = TagDispatch(msg, ptr, ctx, table, 0, {}); if (ptr == nullptr) break; if (ctx->LastTag() != 1) break; // Ended on terminating tag } @@ -362,9 +359,9 @@ inline PROTOBUF_ALWAYS_INLINE void InvertPacked(TcFieldData& data) { // Message fields ////////////////////////////////////////////////////////////////////////////// -template -inline PROTOBUF_ALWAYS_INLINE -const char* TcParser::SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL) { +template +inline PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); } @@ -373,74 +370,139 @@ const char* TcParser::SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL) { hasbits |= (uint64_t{1} << data.hasbit_idx()); SyncHasbits(msg, hasbits, table); auto& field = RefAt(msg, data.offset()); - if (field == nullptr) { - const MessageLite* default_instance = - table->field_aux(data.aux_idx())->message_default; - field = default_instance->New(ctx->data().arena); - } - if (group_coding) { - return ctx->ParseGroup(field, ptr, FastDecodeTag(saved_tag)); + + if (aux_is_table) { + const auto* inner_table = table->field_aux(data.aux_idx())->table; + if (field == nullptr) { + field = inner_table->default_instance->New(ctx->data().arena); + } + if (group_coding) { + return ctx->ParseGroup(field, ptr, FastDecodeTag(saved_tag), + inner_table); + } + return ctx->ParseMessage(field, ptr, inner_table); + } else { + if (field == nullptr) { + const MessageLite* default_instance = + table->field_aux(data.aux_idx())->message_default(); + field = default_instance->New(ctx->data().arena); + } + if (group_coding) { + return ctx->ParseGroup(field, ptr, FastDecodeTag(saved_tag)); + } + return ctx->ParseMessage(field, ptr); } - return ctx->ParseMessage(field, ptr); } -const char* TcParser::FastMS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( +const char* TcParser::FastMdS1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastMdS2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastGdS1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastGdS2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastMtS1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -const char* TcParser::FastMS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( +const char* TcParser::FastMtS2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -const char* TcParser::FastGS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( +const char* TcParser::FastGtS1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -const char* TcParser::FastGS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( +const char* TcParser::FastGtS2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -template -inline PROTOBUF_ALWAYS_INLINE -const char* TcParser::RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL) { +template +inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); } auto saved_tag = UnalignedLoad(ptr); ptr += sizeof(TagType); SyncHasbits(msg, hasbits, table); - const MessageLite* default_instance = - table->field_aux(data.aux_idx())->message_default; - auto& field = RefAt(msg, data.offset()); - MessageLite* submsg = - field.Add>(default_instance); - if (group_coding) { - return ctx->ParseGroup(submsg, ptr, FastDecodeTag(saved_tag)); + auto* aux = table->field_aux(data.aux_idx()); + if (aux_is_table) { + auto* inner_table = aux->table; + auto& field = RefAt(msg, data.offset()); + MessageLite* submsg = field.Add>( + inner_table->default_instance); + if (group_coding) { + return ctx->ParseGroup(submsg, ptr, FastDecodeTag(saved_tag), + inner_table); + } + return ctx->ParseMessage(submsg, ptr, inner_table); + } else { + const MessageLite* default_instance = aux->message_default(); + auto& field = RefAt(msg, data.offset()); + MessageLite* submsg = + field.Add>(default_instance); + if (group_coding) { + return ctx->ParseGroup(submsg, ptr, FastDecodeTag(saved_tag)); + } + return ctx->ParseMessage(submsg, ptr); } - return ctx->ParseMessage(submsg, ptr); } -const char* TcParser::FastMR1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( +const char* TcParser::FastMdR1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastMdR2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastGdR1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastGdR2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -const char* TcParser::FastMR2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( +const char* TcParser::FastMtR1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -const char* TcParser::FastGR1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( +const char* TcParser::FastMtR2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -const char* TcParser::FastGR2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( +const char* TcParser::FastGtR1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastGtR2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } @@ -977,10 +1039,6 @@ const char* TcParser::FastZ64P2(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_NOINLINE const char* TcParser::FastUnknownEnumFallback( PROTOBUF_TC_PARAM_DECL) { - (void)msg; - (void)ctx; - (void)hasbits; - // If we know we want to put this field directly into the unknown field set, // then we can skip the call to MiniParse and directly call table->fallback. // However, we first have to update `data` to contain the decoded tag. @@ -1261,20 +1319,14 @@ const char* TcParser::FastUR2(PROTOBUF_TC_PARAM_DECL) { ////////////////////////////////////////////////////////////////////////////// namespace { -inline void SetHas(const TcParseTableBase* table, const FieldEntry& entry, - MessageLite* msg, uint64_t& hasbits) { - int32_t has_idx = entry.has_idx; - if (has_idx < 32) { - hasbits |= uint64_t{1} << has_idx; - } else { - auto* hasblocks = &TcParser::RefAt(msg, table->has_bits_offset); +inline void SetHas(const FieldEntry& entry, MessageLite* msg) { + auto has_idx = static_cast(entry.has_idx); #if defined(__x86_64__) && defined(__GNUC__) - asm("bts %1, %0\n" : "+m"(*hasblocks) : "r"(has_idx)); + asm("bts %1, %0\n" : "+m"(*msg) : "r"(has_idx)); #else - auto& hasblock = hasblocks[has_idx / 32]; - hasblock |= uint32_t{1} << (has_idx % 32); + auto& hasblock = TcParser::RefAt(msg, has_idx / 32 * 4); + hasblock |= uint32_t{1} << (has_idx % 32); #endif - } } } // namespace @@ -1364,7 +1416,7 @@ const char* TcParser::MpFixed(PROTOBUF_TC_PARAM_DECL) { } // Set the field present: if (card == field_layout::kFcOptional) { - SetHas(table, entry, msg, hasbits); + SetHas(entry, msg); } else if (card == field_layout::kFcOneof) { ChangeOneof(table, entry, data.tag() >> 3, ctx, msg); } @@ -1501,7 +1553,7 @@ const char* TcParser::MpVarint(PROTOBUF_TC_PARAM_DECL) { // Mark the field as present: const bool is_oneof = card == field_layout::kFcOneof; if (card == field_layout::kFcOptional) { - SetHas(table, entry, msg, hasbits); + SetHas(entry, msg); } else if (is_oneof) { ChangeOneof(table, entry, data.tag() >> 3, ctx, msg); } @@ -1676,7 +1728,7 @@ const char* TcParser::MpString(PROTOBUF_TC_PARAM_DECL) { const bool is_oneof = card == field_layout::kFcOneof; bool need_init = false; if (card == field_layout::kFcOptional) { - SetHas(table, entry, msg, hasbits); + SetHas(entry, msg); } else if (is_oneof) { need_init = ChangeOneof(table, entry, data.tag() >> 3, ctx, msg); } @@ -1789,21 +1841,31 @@ const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) { const bool is_oneof = card == field_layout::kFcOneof; bool need_init = false; if (card == field_layout::kFcOptional) { - SetHas(table, entry, msg, hasbits); + SetHas(entry, msg); } else if (is_oneof) { need_init = ChangeOneof(table, entry, data.tag() >> 3, ctx, msg); } - MessageLite*& field = RefAt(msg, entry.offset); - if (need_init || field == nullptr) { - const MessageLite* default_instance = - table->field_aux(&entry)->message_default; - field = default_instance->New(ctx->data().arena); - } SyncHasbits(msg, hasbits, table); - if (is_group) { - return ctx->ParseGroup(field, ptr, decoded_tag); + MessageLite*& field = RefAt(msg, entry.offset); + if ((type_card & field_layout::kTvMask) == field_layout::kTvTable) { + auto* inner_table = table->field_aux(&entry)->table; + if (need_init || field == nullptr) { + field = inner_table->default_instance->New(ctx->data().arena); + } + if (is_group) { + return ctx->ParseGroup(field, ptr, decoded_tag, inner_table); + } + return ctx->ParseMessage(field, ptr, inner_table); + } else { + if (need_init || field == nullptr) { + field = + table->field_aux(&entry)->message_default()->New(ctx->data().arena); + } + if (is_group) { + return ctx->ParseGroup(field, ptr, decoded_tag); + } + return ctx->ParseMessage(field, ptr); } - return ctx->ParseMessage(field, ptr); } const char* TcParser::MpRepeatedMessage(PROTOBUF_TC_PARAM_DECL) { @@ -1837,15 +1899,24 @@ const char* TcParser::MpRepeatedMessage(PROTOBUF_TC_PARAM_DECL) { } SyncHasbits(msg, hasbits, table); - const MessageLite* default_instance = - table->field_aux(&entry)->message_default; - auto& field = RefAt(msg, entry.offset); - MessageLite* value = - field.Add>(default_instance); - if (is_group) { - return ctx->ParseGroup(value, ptr, decoded_tag); - } - return ctx->ParseMessage(value, ptr); + if ((type_card & field_layout::kTvMask) == field_layout::kTvTable) { + auto* inner_table = table->field_aux(&entry)->table; + auto& field = RefAt(msg, entry.offset); + MessageLite* value = field.Add>( + inner_table->default_instance); + if (is_group) { + return ctx->ParseGroup(value, ptr, decoded_tag, inner_table); + } + return ctx->ParseMessage(value, ptr, inner_table); + } else { + auto& field = RefAt(msg, entry.offset); + MessageLite* value = field.Add>( + table->field_aux(&entry)->message_default()); + if (is_group) { + return ctx->ParseGroup(value, ptr, decoded_tag); + } + return ctx->ParseMessage(value, ptr); + } } const char* TcParser::MpMap(PROTOBUF_TC_PARAM_DECL) { diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index c8fc994f916d0..a8b5eb57be73b 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -109,7 +109,6 @@ #ifndef GOOGLE_PROTOBUF_IO_CODED_STREAM_H__ #define GOOGLE_PROTOBUF_IO_CODED_STREAM_H__ - #include #include diff --git a/src/google/protobuf/io/gzip_stream.h b/src/google/protobuf/io/gzip_stream.h index 4cf71b6c321bb..539f77bd3aabb 100644 --- a/src/google/protobuf/io/gzip_stream.h +++ b/src/google/protobuf/io/gzip_stream.h @@ -43,7 +43,6 @@ #ifndef GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__ #define GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__ - #include #include #include diff --git a/src/google/protobuf/io/printer.h b/src/google/protobuf/io/printer.h index 92a4321c0411e..e30bfa6908b7d 100644 --- a/src/google/protobuf/io/printer.h +++ b/src/google/protobuf/io/printer.h @@ -37,7 +37,6 @@ #ifndef GOOGLE_PROTOBUF_IO_PRINTER_H__ #define GOOGLE_PROTOBUF_IO_PRINTER_H__ - #include #include #include diff --git a/src/google/protobuf/io/tokenizer.h b/src/google/protobuf/io/tokenizer.h index 4abab7e30c5e8..ca2747b4ba2af 100644 --- a/src/google/protobuf/io/tokenizer.h +++ b/src/google/protobuf/io/tokenizer.h @@ -37,7 +37,6 @@ #ifndef GOOGLE_PROTOBUF_IO_TOKENIZER_H__ #define GOOGLE_PROTOBUF_IO_TOKENIZER_H__ - #include #include diff --git a/src/google/protobuf/io/zero_copy_stream.h b/src/google/protobuf/io/zero_copy_stream.h index 2041cbf648fc5..f39267d4658c9 100644 --- a/src/google/protobuf/io/zero_copy_stream.h +++ b/src/google/protobuf/io/zero_copy_stream.h @@ -107,7 +107,6 @@ #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__ #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__ - #include diff --git a/src/google/protobuf/io/zero_copy_stream_impl.h b/src/google/protobuf/io/zero_copy_stream_impl.h index a385992f20e28..04978d18b71b1 100644 --- a/src/google/protobuf/io/zero_copy_stream_impl.h +++ b/src/google/protobuf/io/zero_copy_stream_impl.h @@ -40,7 +40,6 @@ #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_H__ #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_H__ - #include #include diff --git a/src/google/protobuf/io/zero_copy_stream_impl_lite.cc b/src/google/protobuf/io/zero_copy_stream_impl_lite.cc index b3dfd84c7fa14..bea35c1238286 100644 --- a/src/google/protobuf/io/zero_copy_stream_impl_lite.cc +++ b/src/google/protobuf/io/zero_copy_stream_impl_lite.cc @@ -36,12 +36,16 @@ #include #include +#include #include #include #include #include +// Must be included last +#include + namespace google { namespace protobuf { namespace io { diff --git a/src/google/protobuf/io/zero_copy_stream_impl_lite.h b/src/google/protobuf/io/zero_copy_stream_impl_lite.h index cbda32871e8bd..557993907d763 100644 --- a/src/google/protobuf/io/zero_copy_stream_impl_lite.h +++ b/src/google/protobuf/io/zero_copy_stream_impl_lite.h @@ -44,10 +44,10 @@ #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__ #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__ - #include #include #include +#include #include #include @@ -78,6 +78,10 @@ class PROTOBUF_EXPORT ArrayInputStream PROTOBUF_FUTURE_FINAL ArrayInputStream(const void* data, int size, int block_size = -1); ~ArrayInputStream() override = default; + // `ArrayInputStream` is neither copiable nor assignable + ArrayInputStream(const ArrayInputStream&) = delete; + ArrayInputStream& operator=(const ArrayInputStream&) = delete; + // implements ZeroCopyInputStream ---------------------------------- bool Next(const void** data, int* size) override; void BackUp(int count) override; @@ -93,8 +97,6 @@ class PROTOBUF_EXPORT ArrayInputStream PROTOBUF_FUTURE_FINAL int position_; int last_returned_size_; // How many bytes we returned last time Next() // was called (used for error checking only). - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArrayInputStream); }; // =================================================================== @@ -113,6 +115,10 @@ class PROTOBUF_EXPORT ArrayOutputStream PROTOBUF_FUTURE_FINAL ArrayOutputStream(void* data, int size, int block_size = -1); ~ArrayOutputStream() override = default; + // `ArrayOutputStream` is neither copiable nor assignable + ArrayOutputStream(const ArrayOutputStream&) = delete; + ArrayOutputStream& operator=(const ArrayOutputStream&) = delete; + // implements ZeroCopyOutputStream --------------------------------- bool Next(void** data, int* size) override; void BackUp(int count) override; @@ -126,8 +132,6 @@ class PROTOBUF_EXPORT ArrayOutputStream PROTOBUF_FUTURE_FINAL int position_; int last_returned_size_; // How many bytes we returned last time Next() // was called (used for error checking only). - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArrayOutputStream); }; // =================================================================== @@ -148,6 +152,10 @@ class PROTOBUF_EXPORT StringOutputStream PROTOBUF_FUTURE_FINAL explicit StringOutputStream(std::string* target); ~StringOutputStream() override = default; + // `StringOutputStream` is neither copiable nor assignable + StringOutputStream(const StringOutputStream&) = delete; + StringOutputStream& operator=(const StringOutputStream&) = delete; + // implements ZeroCopyOutputStream --------------------------------- bool Next(void** data, int* size) override; void BackUp(int count) override; @@ -157,8 +165,6 @@ class PROTOBUF_EXPORT StringOutputStream PROTOBUF_FUTURE_FINAL static constexpr size_t kMinimumSize = 16; std::string* target_; - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(StringOutputStream); }; // Note: There is no StringInputStream. Instead, just create an @@ -215,6 +221,10 @@ class PROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream { int block_size = -1); ~CopyingInputStreamAdaptor() override; + // `CopyingInputStreamAdaptor` is neither copiable nor assignable + CopyingInputStreamAdaptor(const CopyingInputStreamAdaptor&) = delete; + CopyingInputStreamAdaptor& operator=(const CopyingInputStreamAdaptor&) = delete; + // Call SetOwnsCopyingStream(true) to tell the CopyingInputStreamAdaptor to // delete the underlying CopyingInputStream when it is destroyed. void SetOwnsCopyingStream(bool value) { owns_copying_stream_ = value; } @@ -255,8 +265,6 @@ class PROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream { // BackUp(). These need to be returned again. // 0 <= backup_bytes_ <= buffer_used_ int backup_bytes_; - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingInputStreamAdaptor); }; // =================================================================== @@ -298,6 +306,10 @@ class PROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStream { int block_size = -1); ~CopyingOutputStreamAdaptor() override; + // `CopyingOutputStreamAdaptor` is neither copiable nor assignable + CopyingOutputStreamAdaptor(const CopyingOutputStreamAdaptor&) = delete; + CopyingOutputStreamAdaptor& operator=(const CopyingOutputStreamAdaptor&) = delete; + // Writes all pending data to the underlying stream. Returns false if a // write error occurred on the underlying stream. (The underlying // stream itself is not necessarily flushed.) @@ -342,8 +354,6 @@ class PROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStream { // returned by Next()). When BackUp() is called, we just reduce this. // 0 <= buffer_used_ <= buffer_size_. int buffer_used_; - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingOutputStreamAdaptor); }; // =================================================================== @@ -356,6 +366,10 @@ class PROTOBUF_EXPORT LimitingInputStream PROTOBUF_FUTURE_FINAL LimitingInputStream(ZeroCopyInputStream* input, int64_t limit); ~LimitingInputStream() override; + // `LimitingInputStream` is neither copiable nor assignable + LimitingInputStream(const LimitingInputStream&) = delete; + LimitingInputStream& operator=(const LimitingInputStream&) = delete; + // implements ZeroCopyInputStream ---------------------------------- bool Next(const void** data, int* size) override; void BackUp(int count) override; @@ -367,8 +381,6 @@ class PROTOBUF_EXPORT LimitingInputStream PROTOBUF_FUTURE_FINAL ZeroCopyInputStream* input_; int64_t limit_; // Decreases as we go, becomes negative if we overshoot. int64_t prior_bytes_read_; // Bytes read on underlying stream at construction - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(LimitingInputStream); }; diff --git a/src/google/protobuf/io/zero_copy_stream_unittest.cc b/src/google/protobuf/io/zero_copy_stream_unittest.cc index d4e5b5451499c..5565e38697995 100644 --- a/src/google/protobuf/io/zero_copy_stream_unittest.cc +++ b/src/google/protobuf/io/zero_copy_stream_unittest.cc @@ -61,6 +61,7 @@ #include #include +#include #include #include @@ -570,7 +571,7 @@ TEST_F(IoTest, CompressionOptions) { // Some ad-hoc testing of compression options. std::string golden_filename = - TestUtil::GetTestDataPath("net/proto2/internal/testdata/golden_message"); + TestUtil::GetTestDataPath("third_party/protobuf/testdata/golden_message"); std::string golden; GOOGLE_CHECK_OK(File::GetContents(golden_filename, &golden, true)); diff --git a/src/google/protobuf/map_test.inc b/src/google/protobuf/map_test.inc index 18a7bfb758207..550d986af69c0 100644 --- a/src/google/protobuf/map_test.inc +++ b/src/google/protobuf/map_test.inc @@ -3749,7 +3749,7 @@ TEST(MapSerializationTest, DeterministicSubmessage) { const std::string filename = "golden_message_maps"; std::string golden; GOOGLE_CHECK_OK(File::GetContents( - TestUtil::GetTestDataPath("net/proto2/internal/testdata/" + filename), + TestUtil::GetTestDataPath("third_party/protobuf/testdata/" + filename), &golden, true)); t.ParseFromString(golden); *(p.mutable_m()) = t; @@ -3791,7 +3791,7 @@ TEST(TextFormatMapTest, DynamicMessage) { std::string expected_text; GOOGLE_CHECK_OK( - File::GetContents(TestUtil::GetTestDataPath("net/proto2/internal/" + File::GetContents(TestUtil::GetTestDataPath("third_party/protobuf/" "testdata/map_test_data.txt"), &expected_text, true)); @@ -3808,7 +3808,7 @@ TEST(TextFormatMapTest, Sorted) { std::string expected_text; GOOGLE_CHECK_OK( - File::GetContents(TestUtil::GetTestDataPath("net/proto2/internal/" + File::GetContents(TestUtil::GetTestDataPath("third_party/protobuf/" "testdata/map_test_data.txt"), &expected_text, true)); @@ -3828,10 +3828,10 @@ TEST(TextFormatMapTest, Sorted) { TEST(TextFormatMapTest, ParseCorruptedString) { std::string serialized_message; - GOOGLE_CHECK_OK( - File::GetContents(TestUtil::GetTestDataPath( - "net/proto2/internal/testdata/golden_message_maps"), - &serialized_message, true)); + GOOGLE_CHECK_OK(File::GetContents( + TestUtil::GetTestDataPath( + "third_party/protobuf/testdata/golden_message_maps"), + &serialized_message, true)); UNITTEST::TestMaps message; GOOGLE_CHECK(message.ParseFromString(serialized_message)); TestParseCorruptedString(message); diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index 39ec154c344af..d20963bc675de 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -1197,6 +1197,8 @@ class PROTOBUF_EXPORT Reflection final { void SwapOneofField(Message* lhs, Message* rhs, const OneofDescriptor* oneof_descriptor) const; + void InternalSwap(Message* lhs, Message* rhs) const; + inline bool HasOneofField(const Message& message, const FieldDescriptor* field) const; inline void SetOneofCase(Message* message, diff --git a/src/google/protobuf/message_lite.cc b/src/google/protobuf/message_lite.cc index 3a1b67bf66afc..27a8b38bab563 100644 --- a/src/google/protobuf/message_lite.cc +++ b/src/google/protobuf/message_lite.cc @@ -38,6 +38,7 @@ #include #include #include +#include #include #include diff --git a/src/google/protobuf/message_unittest.inc b/src/google/protobuf/message_unittest.inc index fa2bbbf59974a..19cc9c14860b1 100644 --- a/src/google/protobuf/message_unittest.inc +++ b/src/google/protobuf/message_unittest.inc @@ -124,7 +124,7 @@ TEST(MESSAGE_TEST_NAME, SerializeToBrokenOstream) { TEST(MESSAGE_TEST_NAME, ParseFromFileDescriptor) { std::string filename = - TestUtil::GetTestDataPath("net/proto2/internal/testdata/golden_message"); + TestUtil::GetTestDataPath("third_party/protobuf/testdata/golden_message"); int file = open(filename.c_str(), O_RDONLY | O_BINARY); ASSERT_GE(file, 0); @@ -137,7 +137,7 @@ TEST(MESSAGE_TEST_NAME, ParseFromFileDescriptor) { TEST(MESSAGE_TEST_NAME, ParsePackedFromFileDescriptor) { std::string filename = TestUtil::GetTestDataPath( - "net/proto2/internal/testdata/golden_packed_fields_message"); + "third_party/protobuf/testdata/golden_packed_fields_message"); int file = open(filename.c_str(), O_RDONLY | O_BINARY); ASSERT_GE(file, 0); diff --git a/src/google/protobuf/parse_context.cc b/src/google/protobuf/parse_context.cc index 59852fdb7f3f6..4211babef1a04 100644 --- a/src/google/protobuf/parse_context.cc +++ b/src/google/protobuf/parse_context.cc @@ -279,7 +279,8 @@ const char* ParseContext::ReadSizeAndPushLimitAndDepth(const char* ptr, const char* ParseContext::ParseMessage(MessageLite* msg, const char* ptr) { int old; ptr = ReadSizeAndPushLimitAndDepth(ptr, &old); - ptr = ptr ? msg->_InternalParse(ptr, this) : nullptr; + if (ptr == nullptr) return ptr; + ptr = msg->_InternalParse(ptr, this); depth_++; if (!PopLimit(old)) return nullptr; return ptr; diff --git a/src/google/protobuf/parse_context.h b/src/google/protobuf/parse_context.h index 7aea50cdc385f..96ee320951763 100644 --- a/src/google/protobuf/parse_context.h +++ b/src/google/protobuf/parse_context.h @@ -427,6 +427,17 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream { bool>::type = true> PROTOBUF_NODISCARD const char* ParseMessage(T* msg, const char* ptr); + template + PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char* ParseMessage( + MessageLite* msg, const char* ptr, const Table* table) { + int old; + ptr = ReadSizeAndPushLimitAndDepthInlined(ptr, &old); + ptr = ptr ? TcParser::ParseLoop(msg, ptr, this, table) : nullptr; + depth_++; + if (!PopLimit(old)) return nullptr; + return ptr; + } + template PROTOBUF_NODISCARD PROTOBUF_NDEBUG_INLINE const char* ParseGroup( T* msg, const char* ptr, uint32_t tag) { @@ -439,6 +450,18 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream { return ptr; } + template + PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char* ParseGroup( + MessageLite* msg, const char* ptr, uint32_t tag, const Table* table) { + if (--depth_ < 0) return nullptr; + group_depth_++; + ptr = TcParser::ParseLoop(msg, ptr, this, table); + group_depth_--; + depth_++; + if (PROTOBUF_PREDICT_FALSE(!ConsumeEndGroup(tag))) return nullptr; + return ptr; + } + private: // Out-of-line routine to save space in ParseContext::ParseMessage // int old; @@ -451,6 +474,11 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream { PROTOBUF_NODISCARD const char* ReadSizeAndPushLimitAndDepth(const char* ptr, int* old_limit); + // As above, but fully inlined for the cases where we care about performance + // more than size. eg TcParser. + PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char* + ReadSizeAndPushLimitAndDepthInlined(const char* ptr, int* old_limit); + // The context keeps an internal stack to keep track of the recursive // part of the parse state. // Current depth of the active parser, depth counts down. @@ -749,12 +777,26 @@ PROTOBUF_NODISCARD const char* ParseContext::ParseMessage(T* msg, const char* ptr) { int old; ptr = ReadSizeAndPushLimitAndDepth(ptr, &old); - ptr = ptr ? msg->_InternalParse(ptr, this) : nullptr; + if (ptr == nullptr) return ptr; + ptr = msg->_InternalParse(ptr, this); depth_++; if (!PopLimit(old)) return nullptr; return ptr; } +inline const char* ParseContext::ReadSizeAndPushLimitAndDepthInlined( + const char* ptr, int* old_limit) { + int size = ReadSize(&ptr); + if (PROTOBUF_PREDICT_FALSE(!ptr)) { + // Make sure this isn't uninitialized even on error return + *old_limit = 0; + return nullptr; + } + *old_limit = PushLimit(ptr, size); + if (--depth_ < 0) return nullptr; + return ptr; +} + template const char* EpsCopyInputStream::ReadRepeatedFixed(const char* ptr, Tag expected_tag, diff --git a/src/google/protobuf/stubs/port.h b/src/google/protobuf/stubs/port.h index b074cb1630d79..954132bd5c047 100644 --- a/src/google/protobuf/stubs/port.h +++ b/src/google/protobuf/stubs/port.h @@ -147,57 +147,6 @@ static const int64 kint64min = -kint64max - 1; static const uint32 kuint32max = 0xFFFFFFFFu; static const uint64 kuint64max = uint64_t{0xFFFFFFFFFFFFFFFFu}; -#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) ||\ - defined(MEMORY_SANITIZER) - -#ifdef __cplusplus -extern "C" { -#endif // __cplusplus -uint16_t __sanitizer_unaligned_load16(const void *p); -uint32_t __sanitizer_unaligned_load32(const void *p); -uint64_t __sanitizer_unaligned_load64(const void *p); -void __sanitizer_unaligned_store16(void *p, uint16_t v); -void __sanitizer_unaligned_store32(void *p, uint32_t v); -void __sanitizer_unaligned_store64(void *p, uint64_t v); -#ifdef __cplusplus -} // extern "C" -#endif // __cplusplus - -inline uint16_t GOOGLE_UNALIGNED_LOAD16(const void *p) { - return __sanitizer_unaligned_load16(p); -} - -inline uint32_t GOOGLE_UNALIGNED_LOAD32(const void *p) { - return __sanitizer_unaligned_load32(p); -} - -inline uint64_t GOOGLE_UNALIGNED_LOAD64(const void *p) { - return __sanitizer_unaligned_load64(p); -} - -inline void GOOGLE_UNALIGNED_STORE16(void *p, uint16_t v) { - __sanitizer_unaligned_store16(p, v); -} - -inline void GOOGLE_UNALIGNED_STORE32(void *p, uint32_t v) { - __sanitizer_unaligned_store32(p, v); -} - -inline void GOOGLE_UNALIGNED_STORE64(void *p, uint64_t v) { - __sanitizer_unaligned_store64(p, v); -} - -#elif defined(GOOGLE_PROTOBUF_USE_UNALIGNED) && GOOGLE_PROTOBUF_USE_UNALIGNED - -#define GOOGLE_UNALIGNED_LOAD16(_p) (*reinterpret_cast(_p)) -#define GOOGLE_UNALIGNED_LOAD32(_p) (*reinterpret_cast(_p)) -#define GOOGLE_UNALIGNED_LOAD64(_p) (*reinterpret_cast(_p)) - -#define GOOGLE_UNALIGNED_STORE16(_p, _val) (*reinterpret_cast(_p) = (_val)) -#define GOOGLE_UNALIGNED_STORE32(_p, _val) (*reinterpret_cast(_p) = (_val)) -#define GOOGLE_UNALIGNED_STORE64(_p, _val) (*reinterpret_cast(_p) = (_val)) - -#else inline uint16_t GOOGLE_UNALIGNED_LOAD16(const void *p) { uint16_t t; memcpy(&t, p, sizeof t); @@ -227,7 +176,6 @@ inline void GOOGLE_UNALIGNED_STORE32(void *p, uint32_t v) { inline void GOOGLE_UNALIGNED_STORE64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); } -#endif #if defined(GOOGLE_PROTOBUF_OS_NACL) \ || (defined(__ANDROID__) && defined(__clang__) \ diff --git a/src/google/protobuf/test_util2.h b/src/google/protobuf/test_util2.h index 540af636cbda8..27de06f427ef6 100644 --- a/src/google/protobuf/test_util2.h +++ b/src/google/protobuf/test_util2.h @@ -42,11 +42,17 @@ namespace google { namespace protobuf { namespace TestUtil { -// Translate net/proto2/* -> google/protobuf/* +// Translate net/proto2/* or third_party/protobuf/* to google/protobuf/*. inline std::string TranslatePathToOpensource(const std::string& google3_path) { - const std::string prefix = "net/proto2/"; - GOOGLE_CHECK(google3_path.find(prefix) == 0) << google3_path; - std::string path = google3_path.substr(prefix.size()); + std::string net_proto2 = "net/proto2/"; + std::string third_party_protobuf = "third_party/protobuf/"; + std::string path; + if (google3_path.find(net_proto2) == 0) { + path = google3_path.substr(net_proto2.size()); + } else { + GOOGLE_CHECK(google3_path.find(third_party_protobuf) == 0) << google3_path; + path = google3_path.substr(third_party_protobuf.size()); + } path = StringReplace(path, "internal/", "", false); path = StringReplace(path, "proto/", "", false); diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index 19110499d78b6..2a2cb84f4f71e 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -905,16 +905,18 @@ class TextFormat::Parser::ParserImpl { return true; } if (TryConsume("[")) { - while (true) { - if (!LookingAt("{") && !LookingAt("<")) { - DO(SkipFieldValue()); - } else { - DO(SkipFieldMessage()); - } - if (TryConsume("]")) { - break; + if (!TryConsume("]")) { + while (true) { + if (!LookingAt("{") && !LookingAt("<")) { + DO(SkipFieldValue()); + } else { + DO(SkipFieldMessage()); + } + if (TryConsume("]")) { + break; + } + DO(Consume(",")); } - DO(Consume(",")); } ++recursion_limit_; return true; diff --git a/src/google/protobuf/text_format_unittest.cc b/src/google/protobuf/text_format_unittest.cc index 126f2f3806057..7d0dfdb344cf7 100644 --- a/src/google/protobuf/text_format_unittest.cc +++ b/src/google/protobuf/text_format_unittest.cc @@ -94,7 +94,7 @@ class TextFormatTest : public testing::Test { static void SetUpTestSuite() { GOOGLE_CHECK_OK(File::GetContents( TestUtil::GetTestDataPath( - "net/proto2/internal/" + "third_party/protobuf/" "testdata/text_format_unittest_data_oneof_implemented.txt"), &static_proto_text_format_, true)); CleanStringLineEndings(&static_proto_text_format_, false); @@ -116,7 +116,7 @@ class TextFormatExtensionsTest : public testing::Test { public: static void SetUpTestSuite() { GOOGLE_CHECK_OK(File::GetContents( - TestUtil::GetTestDataPath("net/proto2/internal/testdata/" + TestUtil::GetTestDataPath("third_party/protobuf/testdata/" "text_format_unittest_extensions_data.txt"), &static_proto_text_format_, true)); CleanStringLineEndings(&static_proto_text_format_, false); @@ -1035,6 +1035,19 @@ TEST_F(TextFormatTest, ParseShortRepeatedConcatenatedWithEmpty) { EXPECT_EQ(4, proto_.repeatedgroup(1).a()); } +TEST_F(TextFormatTest, ParseShortRepeatedUnknownEmpty) { + std::string parse_string = + "repeated_string: \"before\"\n" + "unknown_field: []\n" + "repeated_string: \"after\"\n"; + TextFormat::Parser parser; + parser.AllowUnknownField(true); + + ASSERT_TRUE(parser.ParseFromString(parse_string, &proto_)); + + EXPECT_EQ(2, proto_.repeated_string_size()); +} + TEST_F(TextFormatTest, Comments) { // Test that comments are ignored. diff --git a/src/google/protobuf/unittest_mset.proto b/src/google/protobuf/unittest_mset.proto index 32949947a7a91..dd8a3afbf4762 100644 --- a/src/google/protobuf/unittest_mset.proto +++ b/src/google/protobuf/unittest_mset.proto @@ -53,6 +53,11 @@ message NestedTestMessageSetContainer { optional NestedTestMessageSetContainer child = 2; } +message NestedTestInt { + optional fixed32 a = 1; + optional NestedTestInt child = 2; +} + message TestMessageSetExtension1 { extend proto2_wireformat_unittest.TestMessageSet { optional TestMessageSetExtension1 message_set_extension = 1545008; @@ -69,11 +74,6 @@ message TestMessageSetExtension2 { optional string str = 25; } -message NestedTestInt { - optional fixed32 a = 1; - optional NestedTestInt child = 2; -} - message TestMessageSetExtension3 { extend proto2_wireformat_unittest.TestMessageSet { optional TestMessageSetExtension3 message_set_extension = 195273129; diff --git a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc index 96b14db0a5ff4..dc35e51ea2d61 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc @@ -31,8 +31,8 @@ #include #include -#include #include +#include #include #include diff --git a/src/google/protobuf/util/internal/protostream_objectsource_test.cc b/src/google/protobuf/util/internal/protostream_objectsource_test.cc index e16db78a12ca9..95e5f1633eea1 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource_test.cc +++ b/src/google/protobuf/util/internal/protostream_objectsource_test.cc @@ -39,6 +39,10 @@ #include #include #include +#include +#include +#include +#include #include #include #include @@ -46,10 +50,6 @@ #include #include #include -#include -#include -#include -#include #include diff --git a/src/google/protobuf/util/internal/protostream_objectwriter_test.cc b/src/google/protobuf/util/internal/protostream_objectwriter_test.cc index 33489998596c0..24040f4e0d8da 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/protostream_objectwriter_test.cc @@ -40,6 +40,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include #include #include @@ -49,12 +55,6 @@ #include #include #include -#include -#include -#include -#include -#include -#include #include #include #include diff --git a/src/google/protobuf/util/json_util_test.cc b/src/google/protobuf/util/json_util_test.cc index 7216199e8e1b7..ac9408d8cc9dc 100644 --- a/src/google/protobuf/util/json_util_test.cc +++ b/src/google/protobuf/util/json_util_test.cc @@ -34,14 +34,14 @@ #include #include -#include -#include -#include #include #include #include #include #include +#include +#include +#include #include #include diff --git a/src/google/protobuf/util/message_differencer_unittest.cc b/src/google/protobuf/util/message_differencer_unittest.cc index f8e44df407c9d..6e6dc64002f13 100644 --- a/src/google/protobuf/util/message_differencer_unittest.cc +++ b/src/google/protobuf/util/message_differencer_unittest.cc @@ -53,11 +53,11 @@ #include #include #include -#include #include #include #include #include +#include namespace google { namespace protobuf { diff --git a/src/google/protobuf/util/time_util.cc b/src/google/protobuf/util/time_util.cc index 9893aa35df902..ee6406d7a1a0b 100644 --- a/src/google/protobuf/util/time_util.cc +++ b/src/google/protobuf/util/time_util.cc @@ -50,19 +50,24 @@ using google::protobuf::Duration; using google::protobuf::Timestamp; namespace { -static const int kNanosPerSecond = 1000000000; -static const int kMicrosPerSecond = 1000000; -static const int kMillisPerSecond = 1000; -static const int kNanosPerMillisecond = 1000000; -static const int kNanosPerMicrosecond = 1000; -static const int kSecondsPerMinute = 60; // Note that we ignore leap seconds. -static const int kSecondsPerHour = 3600; +static constexpr int32_t kNanosPerSecond = 1000000000; +static constexpr int32_t kMicrosPerSecond = 1000000; +static constexpr int32_t kMillisPerSecond = 1000; +static constexpr int32_t kNanosPerMillisecond = 1000000; +static constexpr int32_t kNanosPerMicrosecond = 1000; +static constexpr int32_t kSecondsPerMinute = + 60; // Note that we ignore leap seconds. +static constexpr int32_t kSecondsPerHour = 3600; template -T CreateNormalized(int64_t seconds, int64_t nanos); +T CreateNormalized(int64_t seconds, int32_t nanos); template <> -Timestamp CreateNormalized(int64_t seconds, int64_t nanos) { +Timestamp CreateNormalized(int64_t seconds, int32_t nanos) { + GOOGLE_DCHECK(seconds >= TimeUtil::kTimestampMinSeconds && + seconds <= TimeUtil::kTimestampMaxSeconds) + << "Timestamp seconds are outside of the valid range"; + // Make sure nanos is in the range. if (nanos <= -kNanosPerSecond || nanos >= kNanosPerSecond) { seconds += nanos / kNanosPerSecond; @@ -73,8 +78,12 @@ Timestamp CreateNormalized(int64_t seconds, int64_t nanos) { seconds -= 1; nanos += kNanosPerSecond; } + GOOGLE_DCHECK(seconds >= TimeUtil::kTimestampMinSeconds && - seconds <= TimeUtil::kTimestampMaxSeconds); + seconds <= TimeUtil::kTimestampMaxSeconds && + nanos >= TimeUtil::kTimestampMinNanoseconds && + nanos <= TimeUtil::kTimestampMaxNanoseconds) + << "Timestamp is outside of the valid range"; Timestamp result; result.set_seconds(seconds); result.set_nanos(static_cast(nanos)); @@ -82,7 +91,11 @@ Timestamp CreateNormalized(int64_t seconds, int64_t nanos) { } template <> -Duration CreateNormalized(int64_t seconds, int64_t nanos) { +Duration CreateNormalized(int64_t seconds, int32_t nanos) { + GOOGLE_DCHECK(seconds >= TimeUtil::kDurationMinSeconds && + seconds <= TimeUtil::kDurationMaxSeconds) + << "Duration seconds are outside of the valid range"; + // Make sure nanos is in the range. if (nanos <= -kNanosPerSecond || nanos >= kNanosPerSecond) { seconds += nanos / kNanosPerSecond; @@ -96,8 +109,12 @@ Duration CreateNormalized(int64_t seconds, int64_t nanos) { seconds -= 1; nanos += kNanosPerSecond; } + GOOGLE_DCHECK(seconds >= TimeUtil::kDurationMinSeconds && - seconds <= TimeUtil::kDurationMaxSeconds); + seconds <= TimeUtil::kDurationMaxSeconds && + nanos >= TimeUtil::kDurationMinNanoseconds && + nanos <= TimeUtil::kDurationMaxNanoseconds) + << "Duration is outside of the valid range"; Duration result; result.set_seconds(seconds); result.set_nanos(static_cast(nanos)); @@ -148,10 +165,14 @@ int64_t RoundTowardZero(int64_t value, int64_t divider) { // Actually define these static const integers. Required by C++ standard (but // some compilers don't like it). #ifndef _MSC_VER -const int64_t TimeUtil::kTimestampMinSeconds; -const int64_t TimeUtil::kTimestampMaxSeconds; -const int64_t TimeUtil::kDurationMaxSeconds; -const int64_t TimeUtil::kDurationMinSeconds; +constexpr int64_t TimeUtil::kTimestampMinSeconds; +constexpr int64_t TimeUtil::kTimestampMaxSeconds; +constexpr int32_t TimeUtil::kTimestampMinNanoseconds; +constexpr int32_t TimeUtil::kTimestampMaxNanoseconds; +constexpr int64_t TimeUtil::kDurationMaxSeconds; +constexpr int64_t TimeUtil::kDurationMinSeconds; +constexpr int32_t TimeUtil::kDurationMaxNanoseconds; +constexpr int32_t TimeUtil::kDurationMinNanoseconds; #endif // !_MSC_VER std::string TimeUtil::ToString(const Timestamp& timestamp) { @@ -261,37 +282,43 @@ Duration TimeUtil::SecondsToDuration(int64_t seconds) { } Duration TimeUtil::MinutesToDuration(int64_t minutes) { - return CreateNormalized(minutes * kSecondsPerMinute, 0); + GOOGLE_DCHECK(minutes >= TimeUtil::kDurationMinSeconds / kSecondsPerMinute && + minutes <= TimeUtil::kDurationMaxSeconds / kSecondsPerMinute) + << "Duration minutes are outside of the valid range"; + return SecondsToDuration(minutes * kSecondsPerMinute); } Duration TimeUtil::HoursToDuration(int64_t hours) { - return CreateNormalized(hours * kSecondsPerHour, 0); + GOOGLE_DCHECK(hours >= TimeUtil::kDurationMinSeconds / kSecondsPerHour && + hours <= TimeUtil::kDurationMaxSeconds / kSecondsPerHour) + << "Duration hours are outside of the valid range"; + return SecondsToDuration(hours * kSecondsPerHour); } int64_t TimeUtil::DurationToNanoseconds(const Duration& duration) { + GOOGLE_DCHECK(IsDurationValid(duration)) << "Duration is outside of the valid range"; return duration.seconds() * kNanosPerSecond + duration.nanos(); } int64_t TimeUtil::DurationToMicroseconds(const Duration& duration) { - return duration.seconds() * kMicrosPerSecond + - RoundTowardZero(duration.nanos(), kNanosPerMicrosecond); + return RoundTowardZero(DurationToNanoseconds(duration), kNanosPerMicrosecond); } int64_t TimeUtil::DurationToMilliseconds(const Duration& duration) { - return duration.seconds() * kMillisPerSecond + - RoundTowardZero(duration.nanos(), kNanosPerMillisecond); + return RoundTowardZero(DurationToNanoseconds(duration), kNanosPerMillisecond); } int64_t TimeUtil::DurationToSeconds(const Duration& duration) { + GOOGLE_DCHECK(IsDurationValid(duration)) << "Duration is outside of the valid range"; return duration.seconds(); } int64_t TimeUtil::DurationToMinutes(const Duration& duration) { - return RoundTowardZero(duration.seconds(), kSecondsPerMinute); + return RoundTowardZero(DurationToSeconds(duration), kSecondsPerMinute); } int64_t TimeUtil::DurationToHours(const Duration& duration) { - return RoundTowardZero(duration.seconds(), kSecondsPerHour); + return RoundTowardZero(DurationToSeconds(duration), kSecondsPerHour); } Timestamp TimeUtil::NanosecondsToTimestamp(int64_t nanos) { @@ -316,20 +343,28 @@ Timestamp TimeUtil::SecondsToTimestamp(int64_t seconds) { } int64_t TimeUtil::TimestampToNanoseconds(const Timestamp& timestamp) { + GOOGLE_DCHECK(IsTimestampValid(timestamp)) + << "Timestamp is outside of the valid range"; return timestamp.seconds() * kNanosPerSecond + timestamp.nanos(); } int64_t TimeUtil::TimestampToMicroseconds(const Timestamp& timestamp) { + GOOGLE_DCHECK(IsTimestampValid(timestamp)) + << "Timestamp is outside of the valid range"; return timestamp.seconds() * kMicrosPerSecond + RoundTowardZero(timestamp.nanos(), kNanosPerMicrosecond); } int64_t TimeUtil::TimestampToMilliseconds(const Timestamp& timestamp) { + GOOGLE_DCHECK(IsTimestampValid(timestamp)) + << "Timestamp is outside of the valid range"; return timestamp.seconds() * kMillisPerSecond + RoundTowardZero(timestamp.nanos(), kNanosPerMillisecond); } int64_t TimeUtil::TimestampToSeconds(const Timestamp& timestamp) { + GOOGLE_DCHECK(IsTimestampValid(timestamp)) + << "Timestamp is outside of the valid range"; return timestamp.seconds(); } diff --git a/src/google/protobuf/util/time_util.h b/src/google/protobuf/util/time_util.h index 709527ea273d0..bc45f70cde0d7 100644 --- a/src/google/protobuf/util/time_util.h +++ b/src/google/protobuf/util/time_util.h @@ -69,11 +69,29 @@ class PROTOBUF_EXPORT TimeUtil { // The min/max Timestamp/Duration values we support. // // For "0001-01-01T00:00:00Z". - static const int64_t kTimestampMinSeconds = -62135596800LL; + static constexpr int64_t kTimestampMinSeconds = -62135596800LL; // For "9999-12-31T23:59:59.999999999Z". - static const int64_t kTimestampMaxSeconds = 253402300799LL; - static const int64_t kDurationMinSeconds = -315576000000LL; - static const int64_t kDurationMaxSeconds = 315576000000LL; + static constexpr int64_t kTimestampMaxSeconds = 253402300799LL; + static constexpr int32_t kTimestampMinNanoseconds = 0; + static constexpr int32_t kTimestampMaxNanoseconds = 999999999; + static constexpr int64_t kDurationMinSeconds = -315576000000LL; + static constexpr int64_t kDurationMaxSeconds = 315576000000LL; + static constexpr int32_t kDurationMinNanoseconds = -999999999; + static constexpr int32_t kDurationMaxNanoseconds = 999999999; + + static bool IsTimestampValid(const Timestamp& timestamp) { + return timestamp.seconds() <= kTimestampMaxSeconds && + timestamp.seconds() >= kTimestampMinSeconds && + timestamp.nanos() <= kTimestampMaxNanoseconds && + timestamp.nanos() >= kTimestampMinNanoseconds; + } + + static bool IsDurationValid(const Duration& duration) { + return duration.seconds() <= kDurationMaxSeconds && + duration.seconds() >= kDurationMinSeconds && + duration.nanos() <= kDurationMaxNanoseconds && + duration.nanos() >= kDurationMinNanoseconds; + } // Converts Timestamp to/from RFC 3339 date string format. // Generated output will always be Z-normalized and uses 3, 6 or 9 diff --git a/src/google/protobuf/util/time_util_test.cc b/src/google/protobuf/util/time_util_test.cc index 79e2427da0eed..a741218c90fa0 100644 --- a/src/google/protobuf/util/time_util_test.cc +++ b/src/google/protobuf/util/time_util_test.cc @@ -378,6 +378,138 @@ TEST(TimeUtilTest, TimestampOperators) { EXPECT_TRUE(t2 != t1); } +TEST(TimeUtilTest, IsDurationValid) { + Duration valid; + Duration overflow; + overflow.set_seconds(TimeUtil::kDurationMaxSeconds + 1); + Duration underflow; + underflow.set_seconds(TimeUtil::kDurationMinSeconds - 1); + Duration overflow_nanos; + overflow_nanos.set_nanos(TimeUtil::kDurationMaxNanoseconds + 1); + Duration underflow_nanos; + underflow_nanos.set_nanos(TimeUtil::kDurationMinNanoseconds - 1); + + EXPECT_TRUE(TimeUtil::IsDurationValid(valid)); + EXPECT_FALSE(TimeUtil::IsDurationValid(overflow)); + EXPECT_FALSE(TimeUtil::IsDurationValid(underflow)); + EXPECT_FALSE(TimeUtil::IsDurationValid(overflow_nanos)); + EXPECT_FALSE(TimeUtil::IsDurationValid(underflow_nanos)); +} + +TEST(TimeUtilTest, IsTimestampValid) { + Timestamp valid; + Timestamp overflow; + overflow.set_seconds(TimeUtil::kTimestampMaxSeconds + 1); + Timestamp underflow; + underflow.set_seconds(TimeUtil::kTimestampMinSeconds - 1); + Timestamp overflow_nanos; + overflow_nanos.set_nanos(TimeUtil::kTimestampMaxNanoseconds + 1); + Timestamp underflow_nanos; + underflow_nanos.set_nanos(TimeUtil::kTimestampMinNanoseconds - 1); + + EXPECT_TRUE(TimeUtil::IsTimestampValid(valid)); + EXPECT_FALSE(TimeUtil::IsTimestampValid(overflow)); + EXPECT_FALSE(TimeUtil::IsTimestampValid(underflow)); + EXPECT_FALSE(TimeUtil::IsTimestampValid(overflow_nanos)); + EXPECT_FALSE(TimeUtil::IsTimestampValid(underflow_nanos)); +} + +#ifdef PROTOBUF_HAS_DEATH_TEST // death tests do not work on Windows yet. + +TEST(TimeUtilTest, DurationBounds) { + Duration overflow; + overflow.set_seconds(TimeUtil::kDurationMaxSeconds + 1); + Duration underflow; + underflow.set_seconds(TimeUtil::kDurationMinSeconds - 1); + Duration overflow_nanos; + overflow_nanos.set_nanos(TimeUtil::kDurationMaxNanoseconds + 1); + Duration underflow_nanos; + underflow_nanos.set_nanos(TimeUtil::kDurationMinNanoseconds - 1); + + EXPECT_DEBUG_DEATH({ TimeUtil::SecondsToDuration(overflow.seconds()); }, + "Duration seconds"); + EXPECT_DEBUG_DEATH({ TimeUtil::SecondsToDuration(underflow.seconds()); }, + "Duration seconds"); + EXPECT_DEBUG_DEATH( + { TimeUtil::MinutesToDuration(overflow.seconds() / 60 + 1); }, + "Duration minutes"); + EXPECT_DEBUG_DEATH( + { TimeUtil::MinutesToDuration(underflow.seconds() / 60 - 1); }, + "Duration minutes"); + EXPECT_DEBUG_DEATH( + { TimeUtil::HoursToDuration(overflow.seconds() / 60 + 1); }, + "Duration hours"); + EXPECT_DEBUG_DEATH( + { TimeUtil::HoursToDuration(underflow.seconds() / 60 - 1); }, + "Duration hours"); + + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToNanoseconds(overflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToNanoseconds(underflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToNanoseconds(overflow_nanos); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToNanoseconds(underflow_nanos); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToSeconds(overflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToSeconds(underflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToSeconds(overflow_nanos); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToSeconds(underflow_nanos); }, + "outside of the valid range"); +} + +TEST(TimeUtilTest, TimestampBounds) { + Timestamp overflow; + overflow.set_seconds(TimeUtil::kDurationMaxSeconds + 1); + Timestamp underflow; + underflow.set_seconds(TimeUtil::kDurationMinSeconds - 1); + Timestamp overflow_nanos; + overflow_nanos.set_nanos(TimeUtil::kDurationMaxNanoseconds + 1); + Timestamp underflow_nanos; + underflow_nanos.set_nanos(TimeUtil::kDurationMinNanoseconds - 1); + + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToNanoseconds(overflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToNanoseconds(underflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToNanoseconds(overflow_nanos); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToNanoseconds(underflow_nanos); }, + "outside of the valid range"); + + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMicroseconds(overflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMicroseconds(underflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMicroseconds(overflow_nanos); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMicroseconds(underflow_nanos); }, + "outside of the valid range"); + + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMilliseconds(overflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMilliseconds(underflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMilliseconds(overflow_nanos); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMilliseconds(underflow_nanos); }, + "outside of the valid range"); + + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToSeconds(overflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToSeconds(underflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToSeconds(overflow_nanos); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToSeconds(underflow_nanos); }, + "outside of the valid range"); +} + +#endif // PROTOBUF_HAS_DEATH_TEST + } // namespace } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/util/type_resolver_util_test.cc b/src/google/protobuf/util/type_resolver_util_test.cc index 2780a3937efdc..f7b29889b5fd0 100644 --- a/src/google/protobuf/util/type_resolver_util_test.cc +++ b/src/google/protobuf/util/type_resolver_util_test.cc @@ -42,10 +42,10 @@ #include #include #include -#include #include #include #include +#include namespace google { namespace protobuf {