diff --git a/Sources/SwiftProtobuf/DoubleParser.swift b/Sources/SwiftProtobuf/DoubleParser.swift index c5bf8f989..6cc706de1 100644 --- a/Sources/SwiftProtobuf/DoubleParser.swift +++ b/Sources/SwiftProtobuf/DoubleParser.swift @@ -18,9 +18,12 @@ import Foundation internal class DoubleParser { // Temporary buffer so we can null-terminate the UTF-8 string // before calling the C standard library to parse it. + // // In theory, JSON writers should be able to represent any IEEE Double // in at most 25 bytes, but many writers will emit more digits than - // necessary, so we size this generously. + // necessary, so we size this generously; but we could still fail to + // parse if someone crafts something really long (especially for + // TextFormat due to overflows (see below)). private var work = UnsafeMutableBufferPointer.allocate(capacity: 128) @@ -49,6 +52,15 @@ internal class DoubleParser { // Fail if strtod() did not consume everything we expected // or if strtod() thought the number was out of range. + // + // NOTE: TextFormat specifically calls out handling for overflows + // for float/double fields: + // https://protobuf.dev/reference/protobuf/textformat-spec/#value + // + // > Overflows are treated as infinity or -infinity. + // + // But the JSON protobuf spec doesn't mention anything: + // https://protobuf.dev/programming-guides/proto3/#json if e != work.baseAddress! + bytes.count || !d.isFinite { return nil } diff --git a/Sources/SwiftProtobuf/TextFormatDecoder.swift b/Sources/SwiftProtobuf/TextFormatDecoder.swift index 3e29a73c9..e96d59c05 100644 --- a/Sources/SwiftProtobuf/TextFormatDecoder.swift +++ b/Sources/SwiftProtobuf/TextFormatDecoder.swift @@ -27,6 +27,10 @@ internal struct TextFormatDecoder: Decoder { private var fieldNameMap: _NameMap? private var messageType: any Message.Type + internal var options: TextFormatDecodingOptions { + return scanner.options + } + internal var complete: Bool { mutating get { return scanner.complete @@ -63,27 +67,17 @@ internal struct TextFormatDecoder: Decoder { } mutating func nextFieldNumber() throws -> Int? { - // Per https://protobuf.dev/reference/protobuf/textformat-spec/#fields, every field can be - // followed by a field separator, so if we've seen a field, remove the separator before - // checking for the terminator. if fieldCount > 0 { scanner.skipOptionalSeparator() } - if let terminator = terminator, - scanner.skipOptionalObjectEnd(terminator) { - return nil - } - if let fieldNumber = try scanner.nextFieldNumber(names: fieldNameMap!, messageType: messageType) { + if let fieldNumber = try scanner.nextFieldNumber(names: fieldNameMap!, + messageType: messageType, + terminator: terminator) { fieldCount += 1 return fieldNumber - } else if terminator == nil { - return nil } else { - // If this decoder is looking for at a terminator, then if the scanner failed to - // find a field number, something went wrong (end of stream). - throw TextFormatDecodingError.truncated + return nil } - } mutating func decodeSingularFloatField(value: inout Float) throws { @@ -559,6 +553,7 @@ internal struct TextFormatDecoder: Decoder { var keyField: KeyType.BaseType? var valueField: ValueType.BaseType? let terminator = try scanner.skipObjectStart() + let ignoreExtensionFields = options.ignoreUnknownExtensionFields while true { if scanner.skipOptionalObjectEnd(terminator) { if let keyField = keyField, let valueField = valueField { @@ -568,14 +563,20 @@ internal struct TextFormatDecoder: Decoder { throw TextFormatDecodingError.malformedText } } - if let key = try scanner.nextKey() { + if let key = try scanner.nextKey(allowExtensions: ignoreExtensionFields) { switch key { case "key", "1": try KeyType.decodeSingular(value: &keyField, from: &self) case "value", "2": try ValueType.decodeSingular(value: &valueField, from: &self) default: - throw TextFormatDecodingError.unknownField + if ignoreExtensionFields && key.hasPrefix("[") { + try scanner.skipUnknownFieldValue() + } else if options.ignoreUnknownFields && !key.hasPrefix("[") { + try scanner.skipUnknownFieldValue() + } else { + throw TextFormatDecodingError.unknownField + } } scanner.skipOptionalSeparator() } else { @@ -608,6 +609,7 @@ internal struct TextFormatDecoder: Decoder { var keyField: KeyType.BaseType? var valueField: ValueType? let terminator = try scanner.skipObjectStart() + let ignoreExtensionFields = options.ignoreUnknownExtensionFields while true { if scanner.skipOptionalObjectEnd(terminator) { if let keyField = keyField, let valueField = valueField { @@ -617,14 +619,20 @@ internal struct TextFormatDecoder: Decoder { throw TextFormatDecodingError.malformedText } } - if let key = try scanner.nextKey() { + if let key = try scanner.nextKey(allowExtensions: ignoreExtensionFields) { switch key { case "key", "1": try KeyType.decodeSingular(value: &keyField, from: &self) case "value", "2": try decodeSingularEnumField(value: &valueField) default: - throw TextFormatDecodingError.unknownField + if ignoreExtensionFields && key.hasPrefix("[") { + try scanner.skipUnknownFieldValue() + } else if options.ignoreUnknownFields && !key.hasPrefix("[") { + try scanner.skipUnknownFieldValue() + } else { + throw TextFormatDecodingError.unknownField + } } scanner.skipOptionalSeparator() } else { @@ -657,6 +665,7 @@ internal struct TextFormatDecoder: Decoder { var keyField: KeyType.BaseType? var valueField: ValueType? let terminator = try scanner.skipObjectStart() + let ignoreExtensionFields = options.ignoreUnknownExtensionFields while true { if scanner.skipOptionalObjectEnd(terminator) { if let keyField = keyField, let valueField = valueField { @@ -666,14 +675,20 @@ internal struct TextFormatDecoder: Decoder { throw TextFormatDecodingError.malformedText } } - if let key = try scanner.nextKey() { + if let key = try scanner.nextKey(allowExtensions: ignoreExtensionFields) { switch key { case "key", "1": try KeyType.decodeSingular(value: &keyField, from: &self) case "value", "2": try decodeSingularMessageField(value: &valueField) default: - throw TextFormatDecodingError.unknownField + if ignoreExtensionFields && key.hasPrefix("[") { + try scanner.skipUnknownFieldValue() + } else if options.ignoreUnknownFields && !key.hasPrefix("[") { + try scanner.skipUnknownFieldValue() + } else { + throw TextFormatDecodingError.unknownField + } } scanner.skipOptionalSeparator() } else { diff --git a/Sources/SwiftProtobuf/TextFormatDecodingOptions.swift b/Sources/SwiftProtobuf/TextFormatDecodingOptions.swift index c56d48f7f..06ede25e4 100644 --- a/Sources/SwiftProtobuf/TextFormatDecodingOptions.swift +++ b/Sources/SwiftProtobuf/TextFormatDecodingOptions.swift @@ -21,5 +21,19 @@ public struct TextFormatDecodingOptions: Sendable { /// while parsing. public var messageDepthLimit: Int = 100 + /// If unknown fields in the TextFormat should be ignored. If they aren't + /// ignored, an error will be raised if one is encountered. + /// + /// Note: This is a lossy option, enabling it means part of the TextFormat + /// is silently skipped. + public var ignoreUnknownFields: Bool = false + + /// If unknown extension fields in the TextFormat should be ignored. If they + /// aren't ignored, an error will be raised if one is encountered. + /// + /// Note: This is a lossy option, enabling it means part of the TextFormat + /// is silently skipped. + public var ignoreUnknownExtensionFields: Bool = false + public init() {} } diff --git a/Sources/SwiftProtobuf/TextFormatScanner.swift b/Sources/SwiftProtobuf/TextFormatScanner.swift index d0e96ff81..b3b2e89ca 100644 --- a/Sources/SwiftProtobuf/TextFormatScanner.swift +++ b/Sources/SwiftProtobuf/TextFormatScanner.swift @@ -237,7 +237,7 @@ internal struct TextFormatScanner { private var end: UnsafeRawPointer private var doubleParser = DoubleParser() - private let options: TextFormatDecodingOptions + internal let options: TextFormatDecodingOptions internal var recursionBudget: Int internal var complete: Bool { @@ -1075,13 +1075,7 @@ internal struct TextFormatScanner { } /// Returns text of next regular key or nil if end-of-input. - /// This considers an extension key [keyname] to be an - /// error, so call nextOptionalExtensionKey first if you - /// want to handle extension keys. - /// - /// This is only used by map parsing; we should be able to - /// rework that to use nextFieldNumber instead. - internal mutating func nextKey() throws -> String? { + internal mutating func nextKey(allowExtensions: Bool) throws -> String? { skipWhitespace() if p == end { return nil @@ -1089,7 +1083,10 @@ internal struct TextFormatScanner { let c = p[0] switch c { case asciiOpenSquareBracket: // [ - throw TextFormatDecodingError.malformedText + if allowExtensions { + return "[\(try parseExtensionKey())]" + } + throw TextFormatDecodingError.unknownField case asciiLowerA...asciiLowerZ, asciiUpperA...asciiUpperZ, asciiOne...asciiNine: // a...z, A...Z, 1...9 @@ -1109,54 +1106,216 @@ internal struct TextFormatScanner { /// /// This function accounts for as much as 2/3 of the total run /// time of the entire parse. - internal mutating func nextFieldNumber(names: _NameMap, messageType: any Message.Type) throws -> Int? { + internal mutating func nextFieldNumber( + names: _NameMap, + messageType: any Message.Type, + terminator: UInt8? + ) throws -> Int? { + while true { + skipWhitespace() + if p == end { + if terminator == nil { + return nil + } else { + // Never got the terminator. + throw TextFormatDecodingError.malformedText + } + } + let c = p[0] + switch c { + case asciiLowerA...asciiLowerZ, + asciiUpperA...asciiUpperZ: // a...z, A...Z + let key = parseUTF8Identifier() + if let fieldNumber = names.number(forProtoName: key) { + return fieldNumber + } + if !options.ignoreUnknownFields { + throw TextFormatDecodingError.unknownField + } + // Unknown field name + break + case asciiOpenSquareBracket: // Start of an extension field + let key = try parseExtensionKey() + if let fieldNumber = extensions?.fieldNumberForProto(messageType: messageType, protoFieldName: key) { + return fieldNumber + } + if !options.ignoreUnknownExtensionFields { + throw TextFormatDecodingError.unknownField + } + // Unknown field name + break + case asciiOne...asciiNine: // 1-9 (field numbers are 123, not 0123) + var fieldNum = Int(c) - Int(asciiZero) + p += 1 + while p != end { + let c = p[0] + if c >= asciiZero && c <= asciiNine { + fieldNum = fieldNum &* 10 &+ (Int(c) - Int(asciiZero)) + } else { + break + } + p += 1 + } + skipWhitespace() + if names.names(for: fieldNum) != nil { + return fieldNum + } + if !options.ignoreUnknownFields { + throw TextFormatDecodingError.unknownField + } + // Unknown field name + break + default: + if c == terminator { + let _ = skipOptionalObjectEnd(c) + return nil + } + throw TextFormatDecodingError.malformedText + } + + assert(options.ignoreUnknownFields || options.ignoreUnknownExtensionFields) + try skipUnknownFieldValue() + // Skip any separator before looping around to try for another field. + skipOptionalSeparator() + } + } + + // Helper to skip past an unknown field value, when called `p` will be pointing + // at the first character after the unknown field name. + internal mutating func skipUnknownFieldValue() throws { + // This is modeled after the C++ text_format.cpp `ConsumeField()` + // + // Guess the type of this field: + // - If this field is not a message, there should be a ":" between the + // field name and the field value and also the field value should not + // start with "{" or "<" which indicates the beginning of a message body. + // - If there is no ":" or there is a "{" or "<" after ":", this field has + // to be a message or the input is ill-formed. + skipWhitespace() - if p == end { - return nil + if (skipOptionalColon()) { + if p == end { + // Nothing after the ':'? + throw TextFormatDecodingError.malformedText + } + let c = p[0] + if c != asciiOpenAngleBracket && c != asciiOpenCurlyBracket { + try skipUnknownPrimativeFieldValue() + } else { + try skipUnknownMessageFieldValue() + } + } else { + try skipUnknownMessageFieldValue() + } + } + + /// Helper to see if this could be the start of a hex or octal number so unknown field + /// value parsing can decide how to parse/validate. + private func isNextNumberFloatingPoint() -> Bool { + // NOTE: If we run out of characters can can't tell, say it isn't and let + // the other code error handle. + var scan = p + var c = scan[0] + + // Floats for decimals can have leading '-' + if c == asciiMinus { + scan += 1 + if scan == end { return false } + c = scan[0] } + + if c == asciiPeriod { + return true // "(-)." : clearly a float + } + + if c == asciiZero { + scan += 1 + if scan == end { return false } // "(-)0[end]" : call it not a float + c = scan[0] + if c == asciiLowerX || // "(-)0x" : hex - not a float + (c >= asciiZero && c <= asciiSeven) { // "(-)0[0-7]" : octal - not a float + return false + } + if c == asciiPeriod { + return true // "(-)0." : clearly a float + } + } + + // At this point, it doesn't realy matter what comes next. We'll call it a floating + // point value since even if it was a decimal, it might be too large for a UInt64 but + // would still be valid for a float/double field. + return true + } + + private mutating func skipUnknownPrimativeFieldValue() throws { + // This is modeled after the C++ text_format.cpp `SkipFieldValue()` let c = p[0] - switch c { - case asciiLowerA...asciiLowerZ, - asciiUpperA...asciiUpperZ: // a...z, A...Z - let key = parseUTF8Identifier() - if let fieldNumber = names.number(forProtoName: key) { - return fieldNumber - } else { - throw TextFormatDecodingError.unknownField + + if c == asciiSingleQuote || c == asciiDoubleQuote { + // Note: the field could be 'bytes', so we can't parse that as a string + // as it might fail. + let _ = try nextBytesValue() + return + } + + if skipOptionalBeginArray() { + if skipOptionalEndArray() { + return + } + while true { + if p == end { + throw TextFormatDecodingError.malformedText + } + let c = p[0] + if c != asciiOpenAngleBracket && c != asciiOpenCurlyBracket { + try skipUnknownPrimativeFieldValue() + } else { + try skipUnknownMessageFieldValue() + } + if skipOptionalEndArray() { + return + } + try skipRequiredComma() } - case asciiOpenSquareBracket: // Start of an extension field - let key = try parseExtensionKey() - if let fieldNumber = extensions?.fieldNumberForProto(messageType: messageType, protoFieldName: key) { - return fieldNumber + } + + // This will also cover "true", "false" for booleans, "nan" for floats. + if let _ = try nextOptionalEnumName() { + skipWhitespace() // `nextOptionalEnumName()` doesn't skip trailing whitespace + return + } + // The above will handing "inf", but this is needed for "-inf". + if let _ = skipOptionalInfinity() { + return + } + + if isNextNumberFloatingPoint() { + let _ = try nextDouble() + } else { + if c == asciiMinus { + let _ = try nextUInt() } else { - throw TextFormatDecodingError.unknownField + let _ = try nextSInt() } - case asciiOne...asciiNine: // 1-9 (field numbers are 123, not 0123) - var fieldNum = Int(c) - Int(asciiZero) - p += 1 - while p != end { - let c = p[0] - if c >= asciiZero && c <= asciiNine { - fieldNum = fieldNum &* 10 &+ (Int(c) - Int(asciiZero)) - } else { - break - } - p += 1 + } + } + + private mutating func skipUnknownMessageFieldValue() throws { + // This is modeled after the C++ text_format.cpp `SkipFieldMessage()` + + let terminator = try skipObjectStart() + while !skipOptionalObjectEnd(terminator) { + if p == end { + throw TextFormatDecodingError.malformedText } - skipWhitespace() - if names.names(for: fieldNum) != nil { - return fieldNum + if let _ = try nextKey(allowExtensions: true) { + // Got a valid field name or extension name ("[ext.name]") } else { - // It was a number that isn't a known field. - // The C++ version (TextFormat::Parser::ParserImpl::ConsumeField()), - // supports an option to file or skip the field's value (this is true - // of unknown names or numbers). - throw TextFormatDecodingError.unknownField + throw TextFormatDecodingError.malformedText } - default: - break + try skipUnknownFieldValue() + skipOptionalSeparator() } - throw TextFormatDecodingError.malformedText } private mutating func skipRequiredCharacter(_ c: UInt8) throws { diff --git a/Tests/SwiftProtobufTests/Test_TextFormatDecodingOptions.swift b/Tests/SwiftProtobufTests/Test_TextFormatDecodingOptions.swift index fba5c37a5..548a535cc 100644 --- a/Tests/SwiftProtobufTests/Test_TextFormatDecodingOptions.swift +++ b/Tests/SwiftProtobufTests/Test_TextFormatDecodingOptions.swift @@ -50,4 +50,615 @@ final class Test_TextFormatDecodingOptions: XCTestCase { } } + // MARK: Ignoring unknown fields + + func testIgnoreUnknown_Fields() throws { + let textInputField = "a:1\noptional_int32: 2\nfoo_bar_baz: 3" + let textInputExtField = "[ext.field]: 1\noptional_int32: 2\n[other_ext]: 3" + + var options = TextFormatDecodingOptions() + options.ignoreUnknownFields = true + + let msg = try SwiftProtoTesting_TestAllTypes(textFormatString: textInputField, options: options) // Shouldn't fail + XCTAssertEqual(msg.textFormatString(), "optional_int32: 2\n") + + do { + let _ = try SwiftProtoTesting_TestAllTypes(textFormatString: textInputExtField, options: options) + XCTFail("Shouldn't get here") + } catch TextFormatDecodingError.unknownField { + // This is what should have happened. + } + } + + func testIgnoreUnknown_ExtensionFields() throws { + let textInputField = "a:1\noptional_int32: 2\nfoo_bar_baz: 3" + let textInputExtField = "[ext.field]: 1\noptional_int32: 2\n[other_ext]: 3" + + var options = TextFormatDecodingOptions() + options.ignoreUnknownExtensionFields = true + + do { + let _ = try SwiftProtoTesting_TestAllTypes(textFormatString: textInputField, options: options) + XCTFail("Shouldn't get here") + } catch TextFormatDecodingError.unknownField { + // This is what should have happened. + } + + let msg = try SwiftProtoTesting_TestAllTypes(textFormatString: textInputExtField, options: options) // Shouldn't fail + XCTAssertEqual(msg.textFormatString(), "optional_int32: 2\n") + } + + func testIgnoreUnknown_Both() throws { + let textInput = "a:1\noptional_int32: 2\n[ext.field]: 3" + + var options = TextFormatDecodingOptions() + options.ignoreUnknownFields = true + options.ignoreUnknownExtensionFields = true + + let msg = try SwiftProtoTesting_TestAllTypes(textFormatString: textInput, options: options) // Shouldn't fail + XCTAssertEqual(msg.textFormatString(), "optional_int32: 2\n") + } + + private struct FieldModes: OptionSet { + let rawValue: Int + + static let single = FieldModes(rawValue: 1 << 0) + static let repeated = FieldModes(rawValue: 1 << 1) + + static let all: FieldModes = [.single, .repeated] + } + + // Custom assert that confirms something parsed as a know field on a message passes and also + // parses when skipped for an unknown field (when the option is enabled). + private func assertDecodeIgnoringUnknownsSucceeds( + _ field: String, + _ value: String, + includeColon: Bool = true, + fieldModes: FieldModes = .all, + file: XCTestFileArgType = #file, + line: UInt = #line + ) { + assert(!fieldModes.isEmpty) + let maybeColon = includeColon ? ":" : "" + let singleText = "optional_\(field)\(maybeColon) \(value)" + let repeatedText = "repeated_\(field): [ \(value) ]" + // First, make sure it decodes into a message correctly. + if fieldModes.contains(.single) { + do { + let msg = try SwiftProtoTesting_TestAllTypes(textFormatString: singleText) + XCTAssertFalse(msg.textFormatString().isEmpty, file: file, line: line) // Should have set some field + } catch { + XCTFail("Shoudn't have failed to decode: \(singleText) - \(error)", file: file, line: line) + } + } + if fieldModes.contains(.repeated) { + do { + let msg = try SwiftProtoTesting_TestAllTypes(textFormatString: repeatedText) + // If there was a value, this something should be set (backdoor to testing + // repeated empty arrays) + XCTAssertEqual(value.isEmpty, msg.textFormatString().isEmpty, file: file, line: line) + } catch { + XCTFail("Shoudn't have failed to decode: \(repeatedText) - \(error)", file: file, line: line) + } + } + + var options = TextFormatDecodingOptions() + options.ignoreUnknownFields = true + options.ignoreUnknownExtensionFields = true + + func assertEmptyDecodeSucceeds(_ text: String) { + do { + let msg = try SwiftProtoTesting_TestEmptyMessage(textFormatString: text, options: options) + XCTAssertTrue(msg.textFormatString().isEmpty, file: file, line: line) + } catch { + XCTFail("Ignoring unknowns shouldn't failed: \(text) - \(error)", file: file, line: line) + } + } + + let singleExtText = "[ext.\(field)]\(maybeColon) \(value)" + let repeatedExtText = "[ext.\(field)]: [ \(value) ]" + + if fieldModes.contains(.single) { + assertEmptyDecodeSucceeds(singleText) + assertEmptyDecodeSucceeds(singleExtText) + assertEmptyDecodeSucceeds("\(singleText) # comment") + assertEmptyDecodeSucceeds("\(singleExtText) # comment") + assertEmptyDecodeSucceeds("unknown_message { \(singleText) }") + assertEmptyDecodeSucceeds("unknown_message { \(singleExtText) }") + assertEmptyDecodeSucceeds("unknown_message {\n # comment before\n \(singleText)\n}") + assertEmptyDecodeSucceeds("unknown_message {\n # comment before\n \(singleExtText)\n}") + assertEmptyDecodeSucceeds("unknown_message {\n \(singleText)\n # comment after\n}") + assertEmptyDecodeSucceeds("unknown_message {\n \(singleExtText)\n # comment after\n}") + assertEmptyDecodeSucceeds("unknown_repeating_message: [ { \(singleText) } ]") + assertEmptyDecodeSucceeds("unknown_repeating_message: [ { \(singleExtText) } ]") + assertEmptyDecodeSucceeds("unknown_repeating_message: [\n # comment before\n { \(singleText) }\n]") + assertEmptyDecodeSucceeds("unknown_repeating_message: [\n # comment before\n { \(singleExtText) }\n]") + assertEmptyDecodeSucceeds("unknown_repeating_message: [\n { \(singleText) }\n # comment after\n]") + assertEmptyDecodeSucceeds("unknown_repeating_message: [\n { \(singleExtText) }\n # comment after\n]") + } + if fieldModes.contains(.repeated) { + assertEmptyDecodeSucceeds(repeatedText) + assertEmptyDecodeSucceeds(repeatedExtText) + assertEmptyDecodeSucceeds("\(repeatedText) # comment after") + assertEmptyDecodeSucceeds("\(repeatedExtText) # comment after") + assertEmptyDecodeSucceeds("unknown_message { \(repeatedText) }") + assertEmptyDecodeSucceeds("unknown_message { \(repeatedExtText) }") + assertEmptyDecodeSucceeds("unknown_message {\n # comment before\n \(repeatedText)\n}") + assertEmptyDecodeSucceeds("unknown_message {\n # comment before\n \(repeatedExtText)\n}") + assertEmptyDecodeSucceeds("unknown_message {\n \(repeatedText)\n # comment after\n}") + assertEmptyDecodeSucceeds("unknown_message {\n \(repeatedExtText)\n # comment after\n}") + assertEmptyDecodeSucceeds("unknown_repeating_message: [ { \(repeatedText) } ]") + assertEmptyDecodeSucceeds("unknown_repeating_message: [ { \(repeatedExtText) } ]") + assertEmptyDecodeSucceeds("unknown_repeating_message: [\n # comment before\n { \(repeatedText) }\n]") + assertEmptyDecodeSucceeds("unknown_repeating_message: [\n # comment before\n { \(repeatedExtText) }\n]") + assertEmptyDecodeSucceeds("unknown_repeating_message: [\n { \(repeatedText) }\n # comment after\n]") + assertEmptyDecodeSucceeds("unknown_repeating_message: [\n { \(repeatedExtText) }\n # comment after\n]") + } + } + + // Custom assert that confirms something parsed as a know field on a message fails and also + // fails when skipped for an unknown field (when the option is enabled). + private func assertDecodeIgnoringUnknownsFails( + _ field: String, + _ value: String, + includeColon: Bool = true, + fieldModes: FieldModes = .all, + file: XCTestFileArgType = #file, + line: UInt = #line + ) { + assert(!fieldModes.isEmpty) + let maybeColon = includeColon ? ":" : "" + let singleText = "optional_\(field)\(maybeColon) \(value)" + let repeatedText = "repeated_\(field)\(maybeColon) [ \(value) ]" + // First, make sure it fails decodes. + if fieldModes.contains(.single) { + do { + let _ = try SwiftProtoTesting_TestAllTypes(textFormatString: singleText) + XCTFail("Should have failed to decode: \(singleText)", file: file, line: line) + } catch { + // Nothing + // TODO: Does it make sense to compare this failure to the ones below? + } + } + if fieldModes.contains(.repeated) { + do { + let _ = try SwiftProtoTesting_TestAllTypes(textFormatString: repeatedText) + XCTFail("Should have failed to decode: \(repeatedText)", file: file, line: line) + } catch { + // Nothing + // TODO: Does it make sense to compare this failure to the ones below? + } + } + + var options = TextFormatDecodingOptions() + options.ignoreUnknownFields = true + options.ignoreUnknownExtensionFields = true + + func assertEmptyDecodeFails(_ text: String) { + do { + let _ = try SwiftProtoTesting_TestEmptyMessage(textFormatString: text, options: options) + XCTFail("Ignoring unknowns should have still failed: \(text)", file: file, line: line) + } catch { + // Nothing + } + } + + let singleExtText = "[ext.\(field)]\(maybeColon) \(value)" + let repeatedExtText = "[ext.\(field)]\(maybeColon) [ \(value) ]" + + // Don't bother with the comment variation as we wouldn't be able to tell if it was + // a failure for the comment or for the field itself. + if fieldModes.contains(.single) { + assertEmptyDecodeFails(singleText) + assertEmptyDecodeFails(singleExtText) + assertEmptyDecodeFails("unknown_message { \(singleText) }") + assertEmptyDecodeFails("unknown_message { \(singleExtText) }") + assertEmptyDecodeFails("unknown_repeating_message: [ { \(singleText) } ]") + assertEmptyDecodeFails("unknown_repeating_message: [ { \(singleExtText) } ]") + } + if fieldModes.contains(.repeated) { + assertEmptyDecodeFails(repeatedText) + assertEmptyDecodeFails(repeatedExtText) + assertEmptyDecodeFails("unknown_message { \(repeatedText) }") + assertEmptyDecodeFails("unknown_message { \(repeatedExtText) }") + assertEmptyDecodeFails("unknown_repeating_message: [ { \(repeatedText) } ]") + assertEmptyDecodeFails("unknown_repeating_message: [ { \(repeatedExtText) } ]") + } + } + + func testIgnoreUnknown_String() { + assertDecodeIgnoringUnknownsSucceeds("string", "'abc'") + assertDecodeIgnoringUnknownsSucceeds("string", "\"abc\"") + assertDecodeIgnoringUnknownsSucceeds("string", "'abc'\n'def'") + assertDecodeIgnoringUnknownsSucceeds("string", "\"abc\"\n\"def\"") + assertDecodeIgnoringUnknownsSucceeds("string", "\" !\\\"#$%&'\"\n") + assertDecodeIgnoringUnknownsSucceeds("string", "\"øùúûüýþÿ\"\n") + assertDecodeIgnoringUnknownsSucceeds("string", "\"\\a\\b\\f\\n\\r\\t\\v\\\"\\'\\\\\\?\"") + assertDecodeIgnoringUnknownsSucceeds("string", "\"\\001\\002\\003\\004\\005\\006\\007\"\n") + assertDecodeIgnoringUnknownsSucceeds("string", "\"\\b\\t\\n\\v\\f\\r\\016\\017\"\n") + assertDecodeIgnoringUnknownsSucceeds("string", "\"\\020\\021\\022\\023\\024\\025\\026\\027\"\n") + assertDecodeIgnoringUnknownsSucceeds("string", "\"\\030\\031\\032\\033\\034\\035\\036\\037\"\n") + assertDecodeIgnoringUnknownsSucceeds("string", "\"☞\"\n") + assertDecodeIgnoringUnknownsSucceeds("string", "\"\\xE2\\x98\\x9E\"") + assertDecodeIgnoringUnknownsSucceeds("string", "\"\\342\\230\\236\"") + + assertDecodeIgnoringUnknownsFails("string", "\"\\z\"") + assertDecodeIgnoringUnknownsFails("string", "\"hello\'") + assertDecodeIgnoringUnknownsFails("string", "\'hello\"") + assertDecodeIgnoringUnknownsFails("string", "\"hello") + // Can't test invalid UTF-8 because as an unknown parse just as bytes. + } + + func testIgnoreUnknown_Bytes() { + assertDecodeIgnoringUnknownsSucceeds("bytes", "'abc'") + assertDecodeIgnoringUnknownsSucceeds("bytes", "\"abc\"") + assertDecodeIgnoringUnknownsSucceeds("bytes", "'abc'\n'def'") + assertDecodeIgnoringUnknownsSucceeds("bytes", "\"abc\"\n\"def\"") + assertDecodeIgnoringUnknownsSucceeds("bytes", "\" !\\\"#$%&'\"\n") + assertDecodeIgnoringUnknownsSucceeds("bytes", "\"øùúûüýþÿ\"\n") + assertDecodeIgnoringUnknownsSucceeds("bytes", "\"\\a\\b\\f\\n\\r\\t\\v\\\"\\'\\\\\\?\"") + assertDecodeIgnoringUnknownsSucceeds("bytes", "\"\\001\\002\\003\\004\\005\\006\\007\"\n") + assertDecodeIgnoringUnknownsSucceeds("bytes", "\"\\b\\t\\n\\v\\f\\r\\016\\017\"\n") + assertDecodeIgnoringUnknownsSucceeds("bytes", "\"\\020\\021\\022\\023\\024\\025\\026\\027\"\n") + assertDecodeIgnoringUnknownsSucceeds("bytes", "\"\\030\\031\\032\\033\\034\\035\\036\\037\"\n") + assertDecodeIgnoringUnknownsSucceeds("bytes", "\"☞\"\n") + assertDecodeIgnoringUnknownsSucceeds("bytes", "\"\\xE2\\x98\\x9E\"") + assertDecodeIgnoringUnknownsSucceeds("bytes", "\"\\342\\230\\236\"") + + assertDecodeIgnoringUnknownsFails("bytes", "\"\\z\"") + assertDecodeIgnoringUnknownsFails("bytes", "\"hello\'") + assertDecodeIgnoringUnknownsFails("bytes", "\'hello\"") + assertDecodeIgnoringUnknownsFails("bytes", "\"hello") + assertDecodeIgnoringUnknownsFails("bytes", "\"\\\"\n") + assertDecodeIgnoringUnknownsFails("bytes", "\"\\x\"\n") + assertDecodeIgnoringUnknownsFails("bytes", "\"\\x&\"\n") + assertDecodeIgnoringUnknownsFails("bytes", "\"\\xg\"\n") + assertDecodeIgnoringUnknownsFails("bytes", "\"\\q\"\n") + assertDecodeIgnoringUnknownsFails("bytes", "\"\\777\"\n") // Out-of-range octal + assertDecodeIgnoringUnknownsFails("bytes", "\"") + assertDecodeIgnoringUnknownsFails("bytes", "\"abcde") + assertDecodeIgnoringUnknownsFails("bytes", "\"\\") + assertDecodeIgnoringUnknownsFails("bytes", "\"\\3") + assertDecodeIgnoringUnknownsFails("bytes", "\"\\32") + assertDecodeIgnoringUnknownsFails("bytes", "\"\\232") + assertDecodeIgnoringUnknownsFails("bytes", "\"\\x") + assertDecodeIgnoringUnknownsFails("bytes", "\"\\x1") + assertDecodeIgnoringUnknownsFails("bytes", "\"\\x12") + assertDecodeIgnoringUnknownsFails("bytes", "\"\\x12q") + } + + func testIgnoreUnknown_Enum() { + assertDecodeIgnoringUnknownsSucceeds("nested_enum", "BAZ") + // Made up values will pass when ignoring unknown fields + } + + func testIgnoreUnknown_Bool() { + assertDecodeIgnoringUnknownsSucceeds("bool", "true") + assertDecodeIgnoringUnknownsSucceeds("bool", "True") + assertDecodeIgnoringUnknownsSucceeds("bool", "t") + assertDecodeIgnoringUnknownsSucceeds("bool", "T") + assertDecodeIgnoringUnknownsSucceeds("bool", "1") + assertDecodeIgnoringUnknownsSucceeds("bool", "false") + assertDecodeIgnoringUnknownsSucceeds("bool", "False") + assertDecodeIgnoringUnknownsSucceeds("bool", "f") + assertDecodeIgnoringUnknownsSucceeds("bool", "F") + assertDecodeIgnoringUnknownsSucceeds("bool", "0") + // Made up values will pass when ignoring unknown fields (as enums) + } + + func testIgnoreUnknown_Integer() { + assertDecodeIgnoringUnknownsSucceeds("int32", "0") + assertDecodeIgnoringUnknownsSucceeds("int32", "-12") + assertDecodeIgnoringUnknownsSucceeds("int32", "0x20") + assertDecodeIgnoringUnknownsSucceeds("int32", "-0x12") + assertDecodeIgnoringUnknownsSucceeds("int32", "01") + assertDecodeIgnoringUnknownsSucceeds("int32", "0123") + assertDecodeIgnoringUnknownsSucceeds("int32", "-01") + assertDecodeIgnoringUnknownsSucceeds("int32", "-0123") + + // Can't test range values for any ints because they would work as floats + + assertDecodeIgnoringUnknownsFails("int32", "0x1g") + assertDecodeIgnoringUnknownsFails("int32", "0x1a2g") + assertDecodeIgnoringUnknownsFails("int32", "-0x1g") + assertDecodeIgnoringUnknownsFails("int32", "-0x1a2g") + assertDecodeIgnoringUnknownsFails("int32", "09") + assertDecodeIgnoringUnknownsFails("int32", "-09") + assertDecodeIgnoringUnknownsFails("int32", "01a") + assertDecodeIgnoringUnknownsFails("int32", "-01a") + assertDecodeIgnoringUnknownsFails("int32", "0128") + assertDecodeIgnoringUnknownsFails("int32", "-0128") + } + + func testIgnoreUnknown_FloatingPoint() { + assertDecodeIgnoringUnknownsSucceeds("float", "0") + + assertDecodeIgnoringUnknownsSucceeds("float", "11.0") + assertDecodeIgnoringUnknownsSucceeds("float", "1.0f") + assertDecodeIgnoringUnknownsSucceeds("float", "12f") + assertDecodeIgnoringUnknownsSucceeds("float", "1.0F") + assertDecodeIgnoringUnknownsSucceeds("float", "12F") + assertDecodeIgnoringUnknownsSucceeds("float", "0.1234") + assertDecodeIgnoringUnknownsSucceeds("float", ".123") + assertDecodeIgnoringUnknownsSucceeds("float", "1.5e3") + assertDecodeIgnoringUnknownsSucceeds("float", "2.5e+3") + assertDecodeIgnoringUnknownsSucceeds("float", "3.5e-3") + + assertDecodeIgnoringUnknownsSucceeds("float", "-11.0") + assertDecodeIgnoringUnknownsSucceeds("float", "-1.0f") + assertDecodeIgnoringUnknownsSucceeds("float", "-12f") + assertDecodeIgnoringUnknownsSucceeds("float", "-1.0F") + assertDecodeIgnoringUnknownsSucceeds("float", "-12F") + assertDecodeIgnoringUnknownsSucceeds("float", "-0.1234") + assertDecodeIgnoringUnknownsSucceeds("float", "-.123") + assertDecodeIgnoringUnknownsSucceeds("float", "-1.5e3") + assertDecodeIgnoringUnknownsSucceeds("float", "-2.5e+3") + assertDecodeIgnoringUnknownsSucceeds("float", "-3.5e-3") + + // This would overload a int, but as a floating point value it will map to "inf". + assertDecodeIgnoringUnknownsSucceeds("float", "999999999999999999999999999999999999") + + assertDecodeIgnoringUnknownsSucceeds("float", "nan") + assertDecodeIgnoringUnknownsSucceeds("float", "inf") + assertDecodeIgnoringUnknownsSucceeds("float", "-inf") + } + + func testIgnoreUnknown_Messages() { + // Both bracing types + assertDecodeIgnoringUnknownsSucceeds("nested_message", "{ bb: 7 }") + assertDecodeIgnoringUnknownsSucceeds("nested_message", "{}") + assertDecodeIgnoringUnknownsSucceeds("nested_message", "< bb: 7 >") + assertDecodeIgnoringUnknownsSucceeds("nested_message", "<>") + // Without the colon after the field name + assertDecodeIgnoringUnknownsSucceeds("nested_message", "{ bb: 7 }", includeColon: false) + assertDecodeIgnoringUnknownsSucceeds("nested_message", "{}", includeColon: false) + assertDecodeIgnoringUnknownsSucceeds("nested_message", "< bb: 7 >", includeColon: false) + assertDecodeIgnoringUnknownsSucceeds("nested_message", "<>", includeColon: false) + + assertDecodeIgnoringUnknownsFails("nested_message", "{ >") + assertDecodeIgnoringUnknownsFails("nested_message", "< }") + assertDecodeIgnoringUnknownsFails("nested_message", "{ bb: 7 >") + assertDecodeIgnoringUnknownsFails("nested_message", "< bb: 7 }") + assertDecodeIgnoringUnknownsFails("nested_message", "{ >", includeColon: false) + assertDecodeIgnoringUnknownsFails("nested_message", "< }", includeColon: false) + assertDecodeIgnoringUnknownsFails("nested_message", "{ bb: 7 >", includeColon: false) + assertDecodeIgnoringUnknownsFails("nested_message", "< bb: 7 }", includeColon: false) + } + + func testIgnoreUnknown_FieldSeparators() { + assertDecodeIgnoringUnknownsSucceeds("foreign_message", "{ c: 1, d: 2 }") + assertDecodeIgnoringUnknownsSucceeds("foreign_message", "{ c: 1; d: 2 }") + assertDecodeIgnoringUnknownsSucceeds("foreign_message", "{ c: 1 d: 2 }") + + // Valid parsing accepts separators after a single field, validate that for unknowns also. + + assertDecodeIgnoringUnknownsSucceeds("string", "'abc',", fieldModes: .single) + assertDecodeIgnoringUnknownsSucceeds("nested_enum", "BAZ,", fieldModes: .single) + assertDecodeIgnoringUnknownsSucceeds("bool", "true,", fieldModes: .single) + assertDecodeIgnoringUnknownsSucceeds("int32", "0,", fieldModes: .single) + assertDecodeIgnoringUnknownsSucceeds("float", "nan,", fieldModes: .single) + assertDecodeIgnoringUnknownsSucceeds("foreign_message", "{ },", fieldModes: .single) + + assertDecodeIgnoringUnknownsSucceeds("string", "'abc';", fieldModes: .single) + assertDecodeIgnoringUnknownsSucceeds("nested_enum", "BAZ;", fieldModes: .single) + assertDecodeIgnoringUnknownsSucceeds("bool", "true;", fieldModes: .single) + assertDecodeIgnoringUnknownsSucceeds("int32", "0;", fieldModes: .single) + assertDecodeIgnoringUnknownsSucceeds("float", "nan;", fieldModes: .single) + assertDecodeIgnoringUnknownsSucceeds("foreign_message", "{ };", fieldModes: .single) + // And now within an a sub message. + assertDecodeIgnoringUnknownsSucceeds("foreign_message", "{ c: 1, }", fieldModes: .single) + assertDecodeIgnoringUnknownsSucceeds("foreign_message", "{ c: 1; }", fieldModes: .single) + + // Extra separators fails. + assertDecodeIgnoringUnknownsFails("string", "'abc',,", fieldModes: .single) + assertDecodeIgnoringUnknownsFails("nested_enum", "BAZ,,", fieldModes: .single) + assertDecodeIgnoringUnknownsFails("bool", "true,,", fieldModes: .single) + assertDecodeIgnoringUnknownsFails("int32", "0,,", fieldModes: .single) + assertDecodeIgnoringUnknownsFails("float", "nan,,", fieldModes: .single) + assertDecodeIgnoringUnknownsFails("foreign_message", "{ },,", fieldModes: .single) + + assertDecodeIgnoringUnknownsFails("string", "'abc';;", fieldModes: .single) + assertDecodeIgnoringUnknownsFails("nested_enum", "BAZ;;", fieldModes: .single) + assertDecodeIgnoringUnknownsFails("bool", "true;;", fieldModes: .single) + assertDecodeIgnoringUnknownsFails("int32", "0;;", fieldModes: .single) + assertDecodeIgnoringUnknownsFails("float", "nan;;", fieldModes: .single) + assertDecodeIgnoringUnknownsFails("foreign_message", "{ };;", fieldModes: .single) + // And now within an a sub message. + assertDecodeIgnoringUnknownsFails("foreign_message", "{ c: 1,, }", fieldModes: .single) + assertDecodeIgnoringUnknownsFails("foreign_message", "{ c: 1;; }", fieldModes: .single) + + // Test a few depths of nesting and separators along the way and unknown fields at the + // start and end of each scope along the way. + + let text = """ + unknown_first_outer: "first", + child { + repeated_child { + unknown_first_inner: [0], + payload { + unknown_first_inner_inner: "test", + optional_int32: 1, + unknown_inner_inner: 2f, + }, + unknown_inner: 3.0, + }, + repeated_child { + unknown_first_inner: 0; + payload { + unknown_first_inner_inner: "test"; + optional_int32: 1; + unknown_inner_inner: 2f; + }, + unknown_inner: [3.0]; + }; + unknown: "nope", + unknown: 12; + }, + unknown_outer: [END]; + unknown_outer_final: "last"; + """ + + var options = TextFormatDecodingOptions() + options.ignoreUnknownFields = true + options.ignoreUnknownExtensionFields = true + + do { + let msg = try SwiftProtoTesting_NestedTestAllTypes(textFormatString: text, options: options) + XCTAssertFalse(msg.textFormatString().isEmpty) + } catch { + XCTFail("Shoudn't have failed to decode: \(error)") + } + + } + + func testIgnoreUnknown_ListSeparators() { + // "repeated_int32: []" - good + assertDecodeIgnoringUnknownsSucceeds("int32", "", fieldModes: .repeated) + + // "repeated_int32: [1 2]" - bad, no commas + assertDecodeIgnoringUnknownsFails("int32", "1 2", fieldModes: .repeated) + // "repeated_int32: [1, 2,]" - bad extra trailing comma with no value + assertDecodeIgnoringUnknownsFails("int32", "1, 2,", fieldModes: .repeated) + } + + func testIgnoreUnknown_Comments() throws { + // Stress test to unknown field parsing deals with comments correctly. + let text = """ + does_not_exist: true # comment + something_else { # comment + # comment + optional_string: "still unknown" + } # comment + + optional_int32: 1 # !!! real field + + does_not_exist: true # comment + something_else { # comment + # comment + optional_string: "still unknown" # comment + optional_string: "still unknown" # comment + # comment + "continued" # comment + # comment + some_int : 0x12 # comment + a_float: #comment + 0.2 # comment + repeat: [ + # comment + -123 # comment + # comment + , # comment + # comment + 0222 # comment + # comment + , # comment + # comment + 012 # comment + # comment + ] # comment + } # comment + + optional_uint32: 2 # !!! real field + + does_not_exist: true # comment + something_else { # comment + # comment + optional_string: "still unknown" + } # comment + + """ + + let expected = SwiftProtoTesting_TestAllTypes.with { + $0.optionalInt32 = 1 + $0.optionalUint32 = 2 + } + + var options = TextFormatDecodingOptions() + options.ignoreUnknownFields = true + + let msg = try SwiftProtoTesting_TestAllTypes(textFormatString: text, options: options) + XCTAssertEqual(expected, msg) + } + + func testIgnoreUnknown_Whitespace() throws { + // Blanket test to unknown field parsing deals with comments correctly. + let text = """ + optional_int32: 1 # !!! real field + + does_not_exist + : + 1 + + something_else { + + optional_string: "still unknown" + + " continued value" + + repeated: [ + 1 , + 0x1 + , + 3, 012 + ] + + } + + repeated_strs: [ + "ab" "cd" , + "de" + , + "xyz" + ] + + an_int:1some_bytes:"abc"msg_field:{a:true}repeated:[1]another_int:3 + + optional_uint32: 2 # !!! real field + """ + + let expected = SwiftProtoTesting_TestAllTypes.with { + $0.optionalInt32 = 1 + $0.optionalUint32 = 2 + } + + var options = TextFormatDecodingOptions() + options.ignoreUnknownFields = true + + let msg = try SwiftProtoTesting_TestAllTypes(textFormatString: text, options: options) + XCTAssertEqual(expected, msg) + } + + func testIgnoreUnknownWithMessageDepthLimit() { + let textInput = "a: { a: { i: 1 } }" + + let tests: [(Int, Bool)] = [ + // Limit, success/failure + ( 10, true ), + ( 4, true ), + ( 3, true ), + ( 2, false ), + ( 1, false ), + ] + + for (limit, expectSuccess) in tests { + do { + var options = TextFormatDecodingOptions() + options.messageDepthLimit = limit + options.ignoreUnknownFields = true + let _ = try SwiftProtoTesting_TestEmptyMessage(textFormatString: textInput, options: options) + if !expectSuccess { + XCTFail("Should not have succeed, limit: \(limit)") + } + } catch TextFormatDecodingError.messageDepthLimit { + if expectSuccess { + XCTFail("Decode failed because of limit, but should *NOT* have, limit: \(limit)") + } else { + // Nothing, this is what was expected. + } + } catch let e { + XCTFail("Decode failed (limit: \(limit) with unexpected error: \(e)") + } + } + } + } diff --git a/Tests/SwiftProtobufTests/Test_TextFormat_Map_proto3.swift b/Tests/SwiftProtobufTests/Test_TextFormat_Map_proto3.swift index a9ba10d60..644530e9c 100644 --- a/Tests/SwiftProtobufTests/Test_TextFormat_Map_proto3.swift +++ b/Tests/SwiftProtobufTests/Test_TextFormat_Map_proto3.swift @@ -136,4 +136,150 @@ final class Test_TextFormat_Map_proto3: XCTestCase, PBTestHelpers { o.mapStringForeignMessage == ["foo": foo] } } + + func test_Int32Int32_ignore_unknown_fields() { + var options = TextFormatDecodingOptions() + options.ignoreUnknownFields = true + + assertTextFormatDecodeSucceeds("map_int32_int32 {\n key: 1\n unknown: 6\n value: 2\n}\n", options: options) {(o: MessageTestType) in + return o.mapInt32Int32 == [1:2] + } + do { + let _ = try MessageTestType(textFormatString: "map_int32_int32 {\n key: 1\n [ext]: 7\n value: 2\n}\n", options: options) + XCTFail("Should have failed") + } catch TextFormatDecodingError.unknownField { + // This is what should have happened. + } catch { + XCTFail("Unexpected error: \(error)") + } + + options.ignoreUnknownFields = false + options.ignoreUnknownExtensionFields = true + + assertTextFormatDecodeSucceeds("map_int32_int32 {\n key: 1\n [ext]: 6\n value: 2\n}\n", options: options) {(o: MessageTestType) in + return o.mapInt32Int32 == [1:2] + } + do { + let _ = try MessageTestType(textFormatString: "map_int32_int32 {\n key: 1\n unknown: 7\n value: 2\n}\n", options: options) + XCTFail("Should have failed") + } catch TextFormatDecodingError.unknownField { + // This is what should have happened. + } catch { + XCTFail("Unexpected error: \(error)") + } + + options.ignoreUnknownFields = true + options.ignoreUnknownExtensionFields = true + + assertTextFormatDecodeSucceeds("map_int32_int32 {\n key: 1\n unknown: 6\n [ext]: 7\n value: 2\n}\n", options: options) {(o: MessageTestType) in + return o.mapInt32Int32 == [1:2] + } + assertTextFormatDecodeSucceeds("map_int32_int32 {unknown: 6, [ext]: 7, key: 1, value: 2}", options: options) {(o: MessageTestType) in + return o.mapInt32Int32 == [1:2] + } + assertTextFormatDecodeSucceeds("map_int32_int32 {key: 1; value: 2; unknown: 6; [ext]: 7}", options: options) {(o: MessageTestType) in + return o.mapInt32Int32 == [1:2] + } + assertTextFormatDecodeSucceeds("map_int32_int32 {key:1 unknown: 6 [ext]: 7 value:2}", options: options) {(o: MessageTestType) in + return o.mapInt32Int32 == [1:2] + } + } + + func test_StringMessage_ignore_unknown_fields() { + var options = TextFormatDecodingOptions() + options.ignoreUnknownFields = true + + let foo = SwiftProtoTesting_ForeignMessage.with {$0.c = 999} + + assertTextFormatDecodeSucceeds("map_string_foreign_message {\n key: \"foo\"\n unknown: 6\n value { c: 999 }\n}\n", options: options) {(o: MessageTestType) in + o.mapStringForeignMessage == ["foo": foo] + } + do { + let _ = try MessageTestType(textFormatString: "map_string_foreign_message {\n key: \"foo\"\n [ext]: 7\n value { c: 999 }\n}\n", options: options) + XCTFail("Should have failed") + } catch TextFormatDecodingError.unknownField { + // This is what should have happened. + } catch { + XCTFail("Unexpected error: \(error)") + } + + options.ignoreUnknownFields = false + options.ignoreUnknownExtensionFields = true + + assertTextFormatDecodeSucceeds("map_string_foreign_message {\n key: \"foo\"\n [ext]: 7\n value { c: 999 }\n}\n", options: options) {(o: MessageTestType) in + o.mapStringForeignMessage == ["foo": foo] + } + do { + let _ = try MessageTestType(textFormatString: "map_string_foreign_message {\n key: \"foo\"\n unknown: 6\n value { c: 999 }\n}\n", options: options) + XCTFail("Should have failed") + } catch TextFormatDecodingError.unknownField { + // This is what should have happened. + } catch { + XCTFail("Unexpected error: \(error)") + } + + options.ignoreUnknownFields = true + options.ignoreUnknownExtensionFields = true + + assertTextFormatDecodeSucceeds("map_string_foreign_message {\n key: \"foo\"\n unknown: 6\n [ext]: 7\n value { c: 999 }\n}\n", options: options) {(o: MessageTestType) in + o.mapStringForeignMessage == ["foo": foo] + } + assertTextFormatDecodeSucceeds("map_string_foreign_message { unknown: 6, [ext]: 7, key: \"foo\", value { c: 999 } }", options: options) {(o: MessageTestType) in + o.mapStringForeignMessage == ["foo": foo] + } + assertTextFormatDecodeSucceeds("map_string_foreign_message { key: \"foo\"; value { c: 999 }; unknown: 6; [ext]: 7 }", options: options) {(o: MessageTestType) in + o.mapStringForeignMessage == ["foo": foo] + } + assertTextFormatDecodeSucceeds("map_string_foreign_message { key: \"foo\" value { c: 999 } unknown: 6 [ext]: 7 }", options: options) {(o: MessageTestType) in + o.mapStringForeignMessage == ["foo": foo] + } + } + + func test_Int32Enum_ignore_unknown_fields() { + var options = TextFormatDecodingOptions() + options.ignoreUnknownFields = true + + assertTextFormatDecodeSucceeds("map_int32_enum {\n key: 1\n unknown: 6\n\n value: MAP_ENUM_BAR\n}\n", options: options) {(o: MessageTestType) in + o.mapInt32Enum == [1: .bar] + } + do { + let _ = try MessageTestType(textFormatString: "map_int32_enum {\n key: 1\n [ext]: 7\n value: MAP_ENUM_BAR\n}\n", options: options) + XCTFail("Should have failed") + } catch TextFormatDecodingError.unknownField { + // This is what should have happened. + } catch { + XCTFail("Unexpected error: \(error)") + } + + options.ignoreUnknownFields = false + options.ignoreUnknownExtensionFields = true + + assertTextFormatDecodeSucceeds("map_int32_enum {\n key: 1\n [ext]: 7\n value: MAP_ENUM_BAR\n}\n", options: options) {(o: MessageTestType) in + o.mapInt32Enum == [1: .bar] + } + do { + let _ = try MessageTestType(textFormatString: "map_int32_enum {\n key: 1\n unknown: 6\n value: MAP_ENUM_BAR\n}\n", options: options) + XCTFail("Should have failed") + } catch TextFormatDecodingError.unknownField { + // This is what should have happened. + } catch { + XCTFail("Unexpected error: \(error)") + } + + options.ignoreUnknownFields = true + options.ignoreUnknownExtensionFields = true + + assertTextFormatDecodeSucceeds("map_int32_enum {\n key: 1\n unknown: 6\n [ext]: 7\n value: MAP_ENUM_BAR\n}\n", options: options) {(o: MessageTestType) in + o.mapInt32Enum == [1: .bar] + } + assertTextFormatDecodeSucceeds("map_int32_enum { unknown: 6, [ext]: 7, key: 1, value: MAP_ENUM_BAR }", options: options) {(o: MessageTestType) in + o.mapInt32Enum == [1: .bar] + } + assertTextFormatDecodeSucceeds("map_int32_enum { key: 1; value: MAP_ENUM_BAR; unknown: 6; [ext]: 7 }", options: options) {(o: MessageTestType) in + o.mapInt32Enum == [1: .bar] + } + assertTextFormatDecodeSucceeds("map_int32_enum { key: 1 value: MAP_ENUM_BAR unknown: 6 [ext]: 7 }", options: options) {(o: MessageTestType) in + o.mapInt32Enum == [1: .bar] + } + } } diff --git a/Tests/SwiftProtobufTests/Test_TextFormat_Unknown.swift b/Tests/SwiftProtobufTests/Test_TextFormat_Unknown.swift index 29ca189ad..394e4d2ea 100644 --- a/Tests/SwiftProtobufTests/Test_TextFormat_Unknown.swift +++ b/Tests/SwiftProtobufTests/Test_TextFormat_Unknown.swift @@ -25,6 +25,13 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { return options } + var decodeIgnoreAllUnknowns: TextFormatDecodingOptions { + var options = TextFormatDecodingOptions() + options.ignoreUnknownFields = true + options.ignoreUnknownExtensionFields = true + return options + } + func test_unknown_varint() throws { let bytes: [UInt8] = [8, 0] let msg = try MessageTestType(serializedBytes: bytes) @@ -38,6 +45,9 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { // This is what should have happened. } + let msg2 = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknowns) // Shouldn't throw + XCTAssertEqual(try msg2.serializedBytes(), []) + let textWithoutUnknowns = msg.textFormatString(options: encodeWithoutUnknowns) XCTAssertEqual(textWithoutUnknowns, "") } @@ -55,6 +65,9 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { // This is what should have happened. } + let msg2 = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknowns) // Shouldn't throw + XCTAssertEqual(try msg2.serializedBytes(), []) + let textWithoutUnknowns = msg.textFormatString(options: encodeWithoutUnknowns) XCTAssertEqual(textWithoutUnknowns, "") } @@ -72,6 +85,9 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { // This is what should have happened. } + let msg2 = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknowns) // Shouldn't throw + XCTAssertEqual(try msg2.serializedBytes(), []) + let textWithoutUnknowns = msg.textFormatString(options: encodeWithoutUnknowns) XCTAssertEqual(textWithoutUnknowns, "") } @@ -90,6 +106,9 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { // This is what should have happened. } + let msg2 = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknowns) // Shouldn't throw + XCTAssertEqual(try msg2.serializedBytes(), []) + let textWithoutUnknowns = msg.textFormatString(options: encodeWithoutUnknowns) XCTAssertEqual(textWithoutUnknowns, "") } @@ -109,6 +128,9 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { // This is what should have happened. } + let msg2 = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknowns) // Shouldn't throw + XCTAssertEqual(try msg2.serializedBytes(), []) + let textWithoutUnknowns = msg.textFormatString(options: encodeWithoutUnknowns) XCTAssertEqual(textWithoutUnknowns, "") } @@ -126,6 +148,9 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { // This is what should have happened. } + let msg2 = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknowns) // Shouldn't throw + XCTAssertEqual(try msg2.serializedBytes(), []) + let textWithoutUnknowns = msg.textFormatString(options: encodeWithoutUnknowns) XCTAssertEqual(textWithoutUnknowns, "") } @@ -144,6 +169,9 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { // This is what should have happened. } + let msg2 = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknowns) // Shouldn't throw + XCTAssertEqual(try msg2.serializedBytes(), []) + let textWithoutUnknowns = msg.textFormatString(options: encodeWithoutUnknowns) XCTAssertEqual(textWithoutUnknowns, "") } @@ -162,6 +190,9 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { // This is what should have happened. } + let msg2 = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknowns) // Shouldn't throw + XCTAssertEqual(try msg2.serializedBytes(), []) + let textWithoutUnknowns = msg.textFormatString(options: encodeWithoutUnknowns) XCTAssertEqual(textWithoutUnknowns, "") } @@ -179,6 +210,9 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { // This is what should have happened. } + let msg2 = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknowns) // Shouldn't throw + XCTAssertEqual(try msg2.serializedBytes(), []) + let textWithoutUnknowns = msg.textFormatString(options: encodeWithoutUnknowns) XCTAssertEqual(textWithoutUnknowns, "") } @@ -218,6 +252,22 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { // This is what should have happened. } + // Since unknowns are limited to a depth of 10, we should be able to since the inner most + // messages are just a string (bytes) blob. + let msg2 = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknowns) // Shouldn't throw + XCTAssertEqual(try msg2.serializedBytes(), []) + + // Since unknowns are limited to depth of 10, lower the depth limit on to confirm we stop + // within the unknowns correctly. + do { + var decodeIgnoreAllUnknownsWithDepthLimit = decodeIgnoreAllUnknowns + decodeIgnoreAllUnknownsWithDepthLimit.messageDepthLimit = 5 + let _ = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknownsWithDepthLimit) + XCTFail("Shouldn't get here") + } catch TextFormatDecodingError.messageDepthLimit { + // This is what should have happened. + } + let textWithoutUnknowns = msg.textFormatString(options: encodeWithoutUnknowns) XCTAssertEqual(textWithoutUnknowns, "") } @@ -235,6 +285,9 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { // This is what should have happened. } + let msg2 = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknowns) // Shouldn't throw + XCTAssertEqual(try msg2.serializedBytes(), []) + let textWithoutUnknowns = msg.textFormatString(options: encodeWithoutUnknowns) XCTAssertEqual(textWithoutUnknowns, "") } @@ -252,6 +305,9 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { // This is what should have happened. } + let msg2 = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknowns) // Shouldn't throw + XCTAssertEqual(try msg2.serializedBytes(), []) + let textWithoutUnknowns = msg.textFormatString(options: encodeWithoutUnknowns) XCTAssertEqual(textWithoutUnknowns, "") } @@ -300,6 +356,13 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { // This is what should have happened. } + do { + let _ = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknowns) + XCTFail("Shouldn't get here") + } catch TextFormatDecodingError.messageDepthLimit { + // This is what should have happened. + } + let textWithoutUnknowns = msg.textFormatString(options: encodeWithoutUnknowns) XCTAssertEqual(textWithoutUnknowns, "") } @@ -317,6 +380,9 @@ final class Test_TextFormat_Unknown: XCTestCase, PBTestHelpers { // This is what should have happened. } + let msg2 = try MessageTestType(textFormatString: text, options: decodeIgnoreAllUnknowns) // Shouldn't throw + XCTAssertEqual(try msg2.serializedBytes(), []) + let textWithoutUnknowns = msg.textFormatString(options: encodeWithoutUnknowns) XCTAssertEqual(textWithoutUnknowns, "") } diff --git a/Tests/SwiftProtobufTests/Test_TextFormat_proto3.swift b/Tests/SwiftProtobufTests/Test_TextFormat_proto3.swift index f4e35ede7..c33f6c9a0 100644 --- a/Tests/SwiftProtobufTests/Test_TextFormat_proto3.swift +++ b/Tests/SwiftProtobufTests/Test_TextFormat_proto3.swift @@ -1304,6 +1304,15 @@ final class Test_TextFormat_proto3: XCTestCase, PBTestHelpers { assertTextFormatDecodeSucceeds("optional_nested_message {bb:7;};") {(o: MessageTestType) in return o.optionalNestedMessage.bb == 7 } + // Make sure duplicate separators fail. + assertTextFormatDecodeFails("optional_int32:1,,") + assertTextFormatDecodeFails("optional_int32:1;;") + assertTextFormatDecodeFails("optional_int32:1,;") + assertTextFormatDecodeFails("optional_int32:1;,") + assertTextFormatDecodeFails("optional_nested_message {bb:7,,}") + assertTextFormatDecodeFails("optional_nested_message {bb:7;;}") + assertTextFormatDecodeFails("optional_nested_message {bb:7,;}") + assertTextFormatDecodeFails("optional_nested_message {bb:7;,}") } //