From e9b114d0f0ca771e2b195cc998fed30c6d3bcdc3 Mon Sep 17 00:00:00 2001 From: Frank Osterfeld Date: Tue, 3 Dec 2024 12:20:53 +0100 Subject: [PATCH] PmtYaml: Parser fixes * Make sure block-style maps in block-style lists are parsed correctly. This is needed to parse our existing GRC files. * Support unexpected whitespace/newlines between key and scalars as well as flow-style maps/lists. Signed-off-by: Frank Osterfeld --- core/include/gnuradio-4.0/YamlPmt.hpp | 97 +++++++++++++-------------- core/test/qa_YamlPmt.cpp | 65 +++++++++++++++++- 2 files changed, 111 insertions(+), 51 deletions(-) diff --git a/core/include/gnuradio-4.0/YamlPmt.hpp b/core/include/gnuradio-4.0/YamlPmt.hpp index 58ad162f2..1c4ba64e4 100644 --- a/core/include/gnuradio-4.0/YamlPmt.hpp +++ b/core/include/gnuradio-4.0/YamlPmt.hpp @@ -288,7 +288,7 @@ struct ParseContext { bool atEndOfDocument() const { return lineIdx == lines.size(); } - std::size_t currentIndent() const { return lines[lineIdx].find_first_not_of(' '); } + std::size_t currentIndent(std::string_view indentChars = " ") const { return lines[lineIdx].find_first_not_of(indentChars); } ParseError makeError(std::string message) const { return {.line = lineIdx + 1, .column = columnIdx + 1, .message = std::move(message)}; } @@ -667,9 +667,16 @@ inline std::expected parsePlainScalar(ParseContext& ctx, } inline std::expected parseScalar(ParseContext& ctx, std::string_view typeTag, int currentIndentLevel) { - // remove leading spaces - ctx.consumeSpaces(); + const auto initialLine = ctx.lineIdx; + ctx.consumeWhitespaceAndComments(); + if (ctx.atEndOfDocument()) { + return std::monostate{}; + } + const auto skippedLines = ctx.lineIdx > initialLine; + if (skippedLines && currentIndentLevel >= 0 && ctx.currentIndent() <= static_cast(currentIndentLevel)) { + return std::monostate{}; + } // handle multi-line indicators '|', '|-', '>', '>-' if ((typeTag == "!!str" || typeTag.empty()) && (!ctx.atEndOfLine() && (ctx.front() == '|' || ctx.front() == '>'))) { char indicator = ctx.front(); @@ -760,32 +767,6 @@ inline std::expected parseScalar(ParseContext& ctx, std:: enum class ValueType { List, Map, Scalar }; -inline ValueType peekToFindValueType(ParseContext ctx, int previousIndent) { - ctx.consumeSpaces(); - if (ctx.startsWith("[")) { - return ValueType::List; - } - if (ctx.startsWith("{")) { - return ValueType::Map; - } - if (!ctx.atEndOfLine()) { - return ValueType::Scalar; - } - ctx.skipToNextLine(); - while (!ctx.atEndOfDocument()) { - ctx.consumeWhitespaceAndComments(); - if (ctx.atEndOfDocument() || (previousIndent >= 0 && ctx.currentIndent() <= static_cast(previousIndent))) { - break; - } - ctx.consumeSpaces(); - if (ctx.startsWith("-")) { - return ValueType::List; - } - return ValueType::Map; - } - return ValueType::Scalar; -} - inline std::expected parseKey(ParseContext& ctx, std::string_view extraDelimiters = {}) { ctx.consumeSpaces(); @@ -823,6 +804,30 @@ inline std::expected parseKey(ParseContext& ctx, std::s return key; } +inline ValueType peekToFindValueType(ParseContext ctx, int previousIndent) { + const auto initialLine = ctx.lineIdx; + ctx.consumeWhitespaceAndComments(); + + if (ctx.atEndOfDocument()) { + return ValueType::Scalar; + } + + const auto skippedLines = ctx.lineIdx > initialLine; + if (skippedLines && previousIndent >= 0 && ctx.currentIndent() <= static_cast(previousIndent)) { + return ValueType::Scalar; + } + if (ctx.startsWith("[") || (ctx.startsWith("- ") || ctx.remainingLine() == "-")) { + return ValueType::List; + } + + if (ctx.startsWith("{")) { + return ValueType::Map; + } + + const auto key = parseKey(ctx); + return key.has_value() ? ValueType::Map : ValueType::Scalar; +} + template struct ConvertList { pmtv::pmt operator()(const std::vector& list) { @@ -931,6 +936,7 @@ inline auto parseFlow(ParseContext& ctx, std::string_view typeTag, int parentInd } inline std::expected parseMap(ParseContext& ctx, int parentIndentLevel) { + ctx.consumeWhitespaceAndComments(); if (ctx.consumeIfStartsWith("{")) { auto map = parseFlow(ctx, "", parentIndentLevel); ctx.skipToNextLine(); @@ -938,25 +944,23 @@ inline std::expected parseMap(ParseContext& ctx, int pa } pmtv::map_t map; + bool firstLine = true; while (!ctx.atEndOfDocument()) { if (ctx.startsWith("---")) { return std::unexpected(ctx.makeError("Parser limitation: Multiple documents not supported")); } - ctx.consumeSpaces(); - if (ctx.atEndOfLine() || ctx.startsWith("#")) { - // skip empty lines and comments - ctx.skipToNextLine(); - continue; - } + ctx.consumeWhitespaceAndComments(); - const auto line_indent = ctx.currentIndent(); + const auto line_indent = firstLine ? ctx.currentIndent(" -") : ctx.currentIndent(); // Ignore "-" if map is in a list if (parentIndentLevel >= 0 && line_indent <= static_cast(parentIndentLevel)) { // indentation decreased; end of current map break; } + firstLine = false; + const auto maybeKey = parseKey(ctx); if (!maybeKey.has_value()) { return std::unexpected(maybeKey.error()); @@ -1009,6 +1013,7 @@ inline std::expected parseMap(ParseContext& ctx, int pa } inline std::expected parseList(ParseContext& ctx, std::string_view typeTag, int parentIndentLevel) { + ctx.consumeWhitespaceAndComments(); if (ctx.consumeIfStartsWith("[")) { auto l = parseFlow(ctx, typeTag, parentIndentLevel); ctx.skipToNextLine(); @@ -1017,15 +1022,8 @@ inline std::expected parseList(ParseContext& ctx, std::st std::vector list; - ctx.skipToNextLine(); - while (!ctx.atEndOfDocument()) { - ctx.consumeSpaces(); - if (ctx.atEndOfLine() || ctx.startsWith("#")) { - // skip empty lines and comments - ctx.skipToNextLine(); - continue; - } + ctx.consumeWhitespaceAndComments(); const std::size_t line_indent = ctx.currentIndent(); if (parentIndentLevel >= 0 && line_indent <= static_cast(parentIndentLevel)) { @@ -1033,16 +1031,15 @@ inline std::expected parseList(ParseContext& ctx, std::st break; } - ctx.consumeSpaces(); - - if (!ctx.consumeIfStartsWith('-')) { + if (!ctx.consumeIfStartsWith("-")) { // not a list item return std::unexpected(ctx.makeError("Expected list item")); } ctx.consumeSpaces(); - const auto tagPos = ctx.columnIdx; + const auto itemIndent = ctx.columnIdx; + const auto maybeLocalTag = parseTag(ctx); if (!maybeLocalTag.has_value()) { return std::unexpected(maybeLocalTag.error()); @@ -1060,7 +1057,7 @@ inline std::expected parseList(ParseContext& ctx, std::st switch (peekedType) { case ValueType::List: { if (!typeTag.empty()) { - return std::unexpected(ctx.makeErrorAtColumn("Cannot have type tag for list containing lists", tagPos)); + return std::unexpected(ctx.makeErrorAtColumn("Cannot have type tag for list containing lists", itemIndent)); } auto parsedValue = parseList(ctx, tag, static_cast(line_indent)); if (!parsedValue.has_value()) { @@ -1071,7 +1068,7 @@ inline std::expected parseList(ParseContext& ctx, std::st } case ValueType::Map: { if (!localTag.empty()) { - return std::unexpected(ctx.makeErrorAtColumn("Cannot have type tag for maps", tagPos)); + return std::unexpected(ctx.makeErrorAtColumn("Cannot have type tag for maps", itemIndent)); } auto parsedValue = parseMap(ctx, static_cast(line_indent)); if (!parsedValue.has_value()) { diff --git a/core/test/qa_YamlPmt.cpp b/core/test/qa_YamlPmt.cpp index 6b4605210..5fe9e96d5 100644 --- a/core/test/qa_YamlPmt.cpp +++ b/core/test/qa_YamlPmt.cpp @@ -119,6 +119,7 @@ void testYAML(std::string_view src, const pmtv::map_t expected) { } else { fmt::println(std::cerr, "Unexpected: {}", formatResult(deserializedMap)); expect(false); + return; } // Then test that serializing and deserializing the map again results in the same map @@ -141,7 +142,7 @@ const boost::ut::suite YamlPmtTests = [] { using namespace std::string_literals; using namespace std::string_view_literals; - "Comments"_test = [] { + "Comments and Whitespace"_test = [] { constexpr std::string_view src1 = R"(# Comment double: !!float64 42 # Comment string: "#Hello" # Comment @@ -149,12 +150,24 @@ null: # Comment #Comment: 43 # string: | # Comment # Hello +number: + 42 +list: + [] +list2: [ 42 +] +map: + {} )"; pmtv::map_t expected; expected["double"] = 42.0; expected["string"] = "#Hello"; expected["null"] = std::monostate{}; + expected["number"] = static_cast(42); + expected["list"] = std::vector{}; + expected["list2"] = std::vector{static_cast(42)}; + expected["map"] = pmtv::map_t{}; testYAML(src1, expected); }; @@ -458,6 +471,8 @@ nullVector: !!null - null emptyVector: !!str [] emptyPmtVector: [] +emptyAfterNewline: + [] flowDouble: !!float64 [1, 2, 3] flowString: !!str ["Hello, ", "World", "Multiple\nlines"] flowMultiline: !!str [ "Hello, " , #] @@ -478,6 +493,14 @@ flowMultiline: !!str [ "Hello, " , #] - !!str [3, 4] - { key: !!str [5, 6] } nestedFlow: [ !!str [1, 2], [3, 4] ] +vectorWithBlockMap: + - name: ArraySink + id: ArraySink + parameters: + name: Block +vectorWithColons: + - "key: value" + - "key2: value2" )"; pmtv::map_t expected; @@ -492,12 +515,15 @@ nestedFlow: [ !!str [1, 2], [3, 4] ] expected["nullVector"] = std::monostate{}; expected["emptyVector"] = std::vector{}; expected["emptyPmtVector"] = std::vector{}; + expected["emptyAfterNewline"] = std::vector{}; expected["flowDouble"] = std::vector{1.0, 2.0, 3.0}; expected["flowString"] = std::vector{"Hello, ", "World", "Multiple\nlines"}; expected["flowMultiline"] = std::vector{"Hello, ", "][", "World", "Multiple\nlines"}; expected["nestedVector"] = std::vector{std::vector{"1", "2"}, std::vector{static_cast(3), static_cast(4)}}; expected["nestedFlow"] = std::vector{std::vector{"1", "2"}, std::vector{static_cast(3), static_cast(4)}}; expected["nestedVector2"] = std::vector{static_cast(42), std::vector{"1", "2"}, std::vector{"3", "4"}, pmtv::map_t{{"key", std::vector{"5", "6"}}}}; + expected["vectorWithBlockMap"] = std::vector{pmtv::map_t{{"name", "ArraySink"}, {"id", "ArraySink"}, {"parameters", pmtv::map_t{{"name", "Block"}}}}}; + expected["vectorWithColons"] = std::vector{"key: value", "key2: value2"}; testYAML(src1, expected); @@ -623,6 +649,43 @@ key#Comment: foo expect(eq(formatResult(yaml::deserialize(invalidKeyComment)), "Error in 3:1: Could not find key/value separator ':'"sv)); }; + "GRC"_test = [] { + constexpr std::string_view src = R"( +blocks: + - name: ArraySink + id: ArraySink + parameters: + name: ArraySink + - name: ArraySource + id: ArraySource + parameters: + name: ArraySource +connections: + - [ArraySource, [0, 0], ArraySink, [1, 1]] + - [ArraySource, [0, 1], ArraySink, [1, 0]] + - [ArraySource, [1, 0], ArraySink, [0, 0]] + - [ArraySource, [1, 1], ArraySink, [0, 1]] +)"; + + pmtv::map_t expected; + pmtv::map_t block1; + block1["name"] = "ArraySink"; + block1["id"] = "ArraySink"; + block1["parameters"] = pmtv::map_t{{"name", "ArraySink"}}; + pmtv::map_t block2; + block2["name"] = "ArraySource"; + block2["id"] = "ArraySource"; + block2["parameters"] = pmtv::map_t{{"name", "ArraySource"}}; + expected["blocks"] = std::vector{block1, block2}; + constexpr auto zero = pmtv::pmt{static_cast(0)}; + constexpr auto one = pmtv::pmt{static_cast(1)}; + using PmtVec = std::vector; + + expected["connections"] = std::vector{PmtVec{"ArraySource", std::vector{zero, zero}, "ArraySink", std::vector{one, one}}, PmtVec{"ArraySource", std::vector{zero, one}, "ArraySink", std::vector{one, zero}}, PmtVec{"ArraySource", std::vector{one, zero}, "ArraySink", std::vector{zero, zero}}, PmtVec{"ArraySource", std::vector{one, one}, "ArraySink", std::vector{zero, one}}}; + + testYAML(src, expected); + }; + "Complex"_test = [] { constexpr std::string_view src = R"( complex: !!complex64 (1.0, -1.0)