diff --git a/ReadMe.md b/ReadMe.md index e2d3410..1f039ab 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -9,18 +9,25 @@ This arduino library is based on [libyaml](https://github.com/yaml/libyaml). -It provides several ways to convert YAML<=>JSON using libyaml, cJSON or ArduinoJson objects. -Supported platforms (some untested): +### Supported platforms: - ESP32 - RP2040 - ESP8266 - SAMD +### Features: +- YAML➔JSON and JSON➔YAML conversion +- ArduinoJson serializers/deserializers +- cJSON serializers/deserializers -### Usage + +---------------------------- + + +## Usage ```cpp #include @@ -34,29 +41,96 @@ or ``` +---------------------------- +## Pure libyaml implementation + +YAML is a superset of JSON, so native conversion from/to JSON is possible without any additional JSON library. + +```cpp + // JSON <=> YAML stream to stream conversion (both ways!), accepts valid JSON or YAML as the input + // Available values for output format: + // YAMLParser::OUTPUT_YAML + // YAMLParser::OUTPUT_JSON + // YAMLParser::OUTPUT_JSON_PRETTY + size_t serializeYml( Stream &source, Stream &destination, OutputFormat_t format ); -#### pure libyaml +``` -YAML is a superset of JSON, so native conversion from JSON is possible without any additional JSON library. +**Convert YAML to JSON** ```cpp -#include - // JSON stream to YAML stream - size_t serializeYml( Stream &json_src_stream, Stream &yml_dest_stream ); + String yaml_str = "hello: world\nboolean: true\nfloat: 1.2345"; + StringStream yaml_stream( yaml_str ); + + serializeYml( yaml_stream, Serial, YAMLParser::OUTPUT_JSON_PRETTY ); + +``` + + + +**Convert JSON to YAML** +```cpp + + String json_str = "{\"hello\": \"world\", \"boolean\": true, \"float\":1.2345}"; + StringStream json_stream( json_str ); + + serializeYml( json_stream, Serial, YAMLParser::OUTPUT_YAML ); + +``` + +---------------------------- + +## Bindings + +ArduinoJson and cJSON bindings operate differently depending on the platform. + +| | ArduinoJson support | cJSON support | +|-----------------|---------------------|------------------------------| +| ESP32 | detected (*) | implicit (built-in esp-idf) | +| ESP8266 | implicit | implicit (bundled) | +| RP2040 | implicit | implicit (bundled) | +| SAMD | implicit | implicit (bundled) | + + +(*) On ESP32 platform, the detection depends on `__has_include()` macro. +So all ArduinoJson functions will be disabled unless `#include ` is found **before** `#include `. + +On ESP8266/RP2040/SAMD platforms it is assumed that ArduinoJson is already available as a dependency. + + +In order to save flash space and/or memory, the defaults bindings can be disabled independently by setting one or all of the +following macros before including ArduinoYaml: + +```cpp +#define YAML_DISABLE_ARDUINOJSON // disable all ArduinoJson functions +#define YAML_DISABLE_CJSON // disable all cJSON functions ``` +Note to self: this should probably be the other way around e.g. explicitely enabled by user. + +Note to readers: should ArduinoJson and/or cJSON be implicitely loaded? +[Feedback is welcome!](https://github.com/tobozo/YAMLDuino/issues) + -#### ArduinoJson +---------------------------- + +## ArduinoJson bindings + +The support is implicitely enabled on most platforms. ```cpp -#include // include this first or functions will be disabled -#include + #include // ESP32 plaforms must include this before ArduinoYaml or functions will be disabled + #include +``` + +Enabling support will expose the following functions: +```cpp // ArduinoJSON object to YAML string size_t serializeYml( JsonVariant src_obj, String &dest_string ); // ArduinoJSON object to YAML stream @@ -72,17 +146,21 @@ YAML is a superset of JSON, so native conversion from JSON is possible without a ``` +---------------------------- +## cJSON bindinds -#### cJSON +The support is implicitely enabled on most platforms and will use the bundled cJSON version. +ESP32 will use the built-in version. -⚠️ cJSON has a memory leak with floats, the leak happens only once though, and may be -avoided by quoting the float, which won't affect yaml output. +⚠️ both versions of cJSON have a memory leak with floats, the leak happens only once though, and +may be avoided by quoting the float, which won't affect yaml output. + + +Enabling support will expose the following functions: ```cpp -// #include // no need to include, cJSON is built-in with esp32 and also bundled with ArduinoYaml -#include // cJSON object to YAML string size_t serializeYml( cJSON* src_obj, String &dest_string ); @@ -95,19 +173,120 @@ avoided by quoting the float, which won't affect yaml output. ``` +---------------------------- + +## String/Stream helper + +Although `const char*` is an acceptable source type for conversion, using `Stream` is recommended as it is more memory efficient. + +The `StringStream` class is provided with this library as a helper. + +```cpp + String my_json = "{\"blah\":true}"; + StringStream json_input_stream(my_json); + + String my_output; + StringStream output_stream(my_output); + +``` + +The `StringStream` bundled class is based on Arduino `String` and can easily be replaced by any class inheriting from `Stream`. + +```cpp +class StringStream : public Stream +{ +public: + StringStream(String &s) : str(s), pos(0) {} + virtual ~StringStream() {}; + virtual int available() { return str.length() - pos; } + virtual int read() { return pos -// #define YAML_DISABLE_ARDUINOJSON -// #define YAML_DISABLE_CJSON +// those defines should always be set *before* including YAMLDuino.h -#include +//#define YAML_DISABLE_ARDUINOJSON +//#define YAML_DISABLE_CJSON +#include // sorry about the notation, but it looks nicer than chunk-splitting+quoting -const char* yaml_example_str = R"_YAML_STRING_( +const char* yaml_sample_str = R"_YAML_STRING_( first: true fourth: false blah: - multiline_string: | + multiline_string_with_trailing_lf: | omg I'm multiline! whelp "I'm quoted" ? ! () { } "\t\r\n" slash\ed array_of_indexed_multiline_strings: - - multiline_string1: | + - with_trailing_lf: | one ! two "?" three ... - - multiline_string2: | + - without_trailing_lf: |- four @'"] five [[(() six +-_%^& @@ -50,25 +51,23 @@ last: true )_YAML_STRING_"; - -const char* json_example_str = R"_JSON_STRING_( +// exact JSON representation of yaml_sample_str +const char* json_sample_str = R"_JSON_STRING_( { "first": true, "fourth": false, "blah": { - "multiline_string": "omg I'm multiline!\nwhelp \"I'm quoted\" ? ! () { } \"\\t\\r\\n\"\nslash\\ed", - "array_of_indexed_multiline_strings": [{"multiline_string1": "one !\ntwo \"?\"\nthree ..."},{"multiline_string2": "four @'\"]\nfive [[(()\nsix +-_%^&"}], + "multiline_string_with_trailing_lf": "omg I'm multiline!\nwhelp \"I'm quoted\" ? ! () { } \"\\t\\r\\n\"\nslash\\ed\n", + "array_of_indexed_multiline_strings": [{ "with_trailing_lf": "one !\ntwo \"?\"\nthree ...\n" }, { "without_trailing_lf": "four @'\"]\nfive [[(()\nsix +-_%^&" } ], "just_a_string": "I am a string", - "array_of_strings": ["oops", "meh" ], + "array_of_strings": [ "oops", "meh" ], "same_array_of_strings": [ "oops", "meh" ], "array_mixed": [ 1, 2, 3, "soleil!" ], - "array_of_anonymous_objects": [ - { + "array_of_anonymous_objects": [{ "prop1": "wizz", "prop2": "pop", "prop3": "snap" - }, - { + }, { "prop1": "foo", "prop3": "bar", "prop2": "baz", @@ -86,11 +85,77 @@ const char* json_example_str = R"_JSON_STRING_( )_JSON_STRING_"; -const size_t yaml_str_size = strlen(yaml_example_str); -const size_t json_str_size = strlen(json_example_str); +const size_t yaml_str_size = strlen(yaml_sample_str); +const size_t json_str_size = strlen(json_sample_str); int test_number = 1; +// The following function is tested in every available format using 'Stream&' as input/output types: +// serializeYml( input, output, format ) + +void test_Yaml2JsonPretty() +{ + YAML_LOG_n( "[TEST #%d] Yaml2Json serializeYml(stream_in, stream_out, YAMLParser::OUTPUT_JSON_PRETTY) using pure libyaml:", test_number++ ); + + String yaml_str = String( yaml_sample_str ); + StringStream yaml_stream( yaml_str ); + + serializeYml( yaml_stream, Serial, YAMLParser::OUTPUT_JSON_PRETTY ); + + YAML_LOG_n("Tests complete"); +} + +void test_Yaml2Json() +{ + YAML_LOG_n( "[TEST #%d] Yaml2Json serializeYml(stream_in, stream_out, YAMLParser::OUTPUT_JSON) using pure libyaml:", test_number++ ); + + String yaml_str = String( yaml_sample_str ); + StringStream yaml_stream( yaml_str ); + + serializeYml( yaml_stream, Serial, YAMLParser::OUTPUT_JSON ); + + YAML_LOG_n("Tests complete"); +} + + +void test_Json2Yaml() +{ + YAML_LOG_n( "[TEST #%d] Json2Yaml serializeYml(stream_in, stream_out, YAMLParser::OUTPUT_YAML using pure libyaml:", test_number++ ); + + String yaml_str = String( yaml_sample_str ); + StringStream yaml_stream( yaml_str ); + + serializeYml( yaml_stream, Serial, YAMLParser::OUTPUT_YAML ); + + YAML_LOG_n("Tests complete"); +} + + + +void test_Readme_Snippet() +{ + YAML_LOG_n( "[TEST #%d] serializeYml() snippets from ReadMe:", test_number++ ); + + String yaml_str = "hello: world\nboolean: true\nfloat: 1.2345\n"; + StringStream yaml_stream( yaml_str ); + serializeYml( yaml_stream, Serial, YAMLParser::OUTPUT_JSON_PRETTY ); + Serial.println(); + + String json_str = "{\"hello\": \"world\", \"boolean\": true, \"float\":1.2345}\n"; + StringStream json_stream( json_str ); + serializeYml( json_stream, Serial, YAMLParser::OUTPUT_YAML ); + Serial.println(); + + YAML_LOG_n("Tests complete"); +} + + + + +// YAML/JSON loading/parsing using YAMLParser object notation +// Note: This decomposition is only here to provide a documented test case. +// Use serializeYml(stream_in, stream_out, format) unless you need to +// manipulate the YAML between import and export. void test_Yaml_String_Parser() { @@ -98,7 +163,7 @@ void test_Yaml_String_Parser() YAMLParser* parser = new YAMLParser(); Stream* output_stream = &Serial; parser->setOutputStream( output_stream ); - parser->parse( json_example_str ); + parser->parse( json_sample_str ); delete parser; YAML_LOG_n("Tests complete"); } @@ -110,7 +175,7 @@ void test_Yaml_Stream_Parser() YAMLParser* parser = new YAMLParser(); Stream* output_stream = &Serial; parser->setOutputStream( output_stream ); - String json_in = String( json_example_str ); + String json_in = String( json_sample_str ); StringStream input_stream( json_in ); parser->parse( input_stream ); delete parser; @@ -124,7 +189,7 @@ void test_Yaml_String_Loader() YAMLParser* parser = new YAMLParser(); Stream* output_stream = &Serial; parser->setOutputStream( output_stream ); - parser->load( json_example_str ); + parser->load( json_sample_str ); // do something with parser->getDocument() parser->parse(); delete parser; @@ -138,7 +203,7 @@ void test_Yaml_Stream_Loader() YAMLParser* parser = new YAMLParser(); Stream* output_stream = &Serial; parser->setOutputStream( output_stream ); - String json_in = String( json_example_str ); + String json_in = String( json_sample_str ); StringStream input_stream( json_in ); parser->load( input_stream ); // do something with parser->getDocument() @@ -150,10 +215,16 @@ void test_Yaml_Stream_Loader() #if defined HAS_ARDUINOJSON + // The following functions are tested using 'const char*' and 'Stream&' as input/output types: + // deserializeYml( JsonObject, input ) + // deserializeYml( JsonDocument, input ) + // serializeYml( JsonObject, output ) + // serializeYml( JsonDocument, output ) + void test_deserializeYml_JsonObject_YamlStream() { YAML_LOG_n( "[TEST #%d] YAML stream to JsonObject -> deserializeYml(json_obj, yaml_stream):", test_number++ ); - String yaml_str = String( yaml_example_str ); + String yaml_str = String( yaml_sample_str ); StringStream yaml_stream( yaml_str ); DynamicJsonDocument json_doc(2048); JsonObject json_obj = json_doc.as(); @@ -171,10 +242,10 @@ void test_Yaml_Stream_Loader() void test_deserializeYml_JsonObject_YamlString() { - YAML_LOG_n( "[TEST #%d] YAML string to JsonObject -> deserializeYml(json_obj, yaml_example_str):", test_number++ ); + YAML_LOG_n( "[TEST #%d] YAML string to JsonObject -> deserializeYml(json_obj, yaml_sample_str):", test_number++ ); DynamicJsonDocument json_doc(2048); JsonObject json_obj = json_doc.as(); - auto err = deserializeYml( json_obj, yaml_example_str ); // deserialize yaml string to JsonObject + auto err = deserializeYml( json_obj, yaml_sample_str ); // deserialize yaml string to JsonObject if( err ) { YAML_LOG_n("Unable to deserialize demo YAML to JsonObject: %s", err.c_str() ); return; @@ -191,7 +262,7 @@ void test_Yaml_Stream_Loader() { YAML_LOG_n( "[TEST #%d] YAML stream to JsonDocument -> deserializeYml(json_doc, yaml_stream):", test_number++ ); DynamicJsonDocument json_doc(2048); - String yaml_str = String( yaml_example_str ); + String yaml_str = String( yaml_sample_str ); StringStream yaml_stream( yaml_str ); auto err = deserializeYml( json_doc, yaml_stream ); // deserialize yaml stream to JsonDocument if( err ) { @@ -207,8 +278,8 @@ void test_Yaml_Stream_Loader() void test_deserializeYml_JsonDocument_YamlString() { - YAML_LOG_n( "[TEST #%d] YAML string to JsonDocument -> deserializeYml(json_doc, yaml_example_str):", test_number++ ); - String yaml_str( yaml_example_str ); + YAML_LOG_n( "[TEST #%d] YAML string to JsonDocument -> deserializeYml(json_doc, yaml_sample_str):", test_number++ ); + String yaml_str( yaml_sample_str ); DynamicJsonDocument json_doc(2048); auto err = deserializeYml( json_doc, yaml_str.c_str() ); // deserialize yaml string to JsonDocument if( err ) { @@ -228,7 +299,7 @@ void test_Yaml_Stream_Loader() // Convert JsonObject to yaml YAML_LOG_n( "[TEST #%d] JsonObject to YAML stream -> serializeYml(json_obj, yaml_stream_out):", test_number++ ); String str_yaml_out = ""; // YAML output string - String json_str = String( json_example_str ); + String json_str = String( json_sample_str ); StringStream yaml_stream_out( str_yaml_out ); // Stream to str_yaml_out DynamicJsonDocument doc(2048); // create and populate a JsonObject auto err = deserializeJson( doc, json_str.c_str() ); @@ -248,7 +319,7 @@ void test_Yaml_Stream_Loader() // Convert JsonObject to yaml YAML_LOG_n( "[TEST #%d] JsonObject to YAML stream -> serializeYml(json_obj, str_yaml_out):", test_number++ ); String str_yaml_out = ""; // YAML output string - String json_str = String( json_example_str ); + String json_str = String( json_sample_str ); DynamicJsonDocument doc(2048); // create and populate a JsonObject auto err = deserializeJson( doc, json_str.c_str() ); if( err ) { @@ -269,12 +340,15 @@ void test_Yaml_Stream_Loader() #if defined HAS_CJSON + // The following functions are tested using 'const char*' and 'Stream&' as input/output types: + // deserializeYml( cJSON*, input ) + // serializeYml( cJSON*, output ) void test_deserializeYml_cJson_String() { - YAML_LOG_n( "[TEST #%d] YAML string to cJSON Object -> deserializeYml(cJSON_obj*, yaml_example_str):", test_number++ ); + YAML_LOG_n( "[TEST #%d] YAML string to cJSON Object -> deserializeYml(cJSON_obj*, yaml_sample_str):", test_number++ ); cJSON* objPtr; - int ret = deserializeYml( &objPtr, yaml_example_str ); // deserialize YAML string into cJSON object + int ret = deserializeYml( &objPtr, yaml_sample_str ); // deserialize YAML string into cJSON object if (ret<0) { Serial.println("deserializeYml failed"); return; @@ -296,7 +370,7 @@ void test_Yaml_Stream_Loader() void test_deserializeYml_cJson_Stream() { YAML_LOG_n( "[TEST #%d] YAML stream to cJSON Object -> deserializeYml(cJSON_obj*, yaml_stream):", test_number++ ); - String yaml_str = String( yaml_example_str ); + String yaml_str = String( yaml_sample_str ); StringStream yaml_stream( yaml_str ); cJSON* objPtr; int ret = deserializeYml( &objPtr, yaml_stream ); // deserialize YAML stream into cJSON object @@ -321,7 +395,7 @@ void test_Yaml_Stream_Loader() void test_serializeYml_cJson_Stream() { YAML_LOG_n( "[TEST #%d] cJSON Object to YAML stream -> serializeYml( objPtr, Serial ):", test_number++ ); - cJSON* objPtr = cJSON_Parse( json_example_str ); + cJSON* objPtr = cJSON_Parse( json_sample_str ); size_t bytes_out = serializeYml( objPtr, Serial ); cJSON_Delete( objPtr ); YAML_LOG_n("[YAML=>cJsonObject=>YAML] yaml bytes in=%d, json bytes out=%d\n", json_str_size, bytes_out); @@ -331,7 +405,7 @@ void test_Yaml_Stream_Loader() void test_serializeYml_cJson_String() { YAML_LOG_n( "[TEST #%d] cJSON Object to YAML string -> serializeYml( objPtr, yaml_dest_str ):", test_number++ ); - cJSON* objPtr = cJSON_Parse( json_example_str ); + cJSON* objPtr = cJSON_Parse( json_sample_str ); String yaml_dest_str; size_t bytes_out = serializeYml( objPtr, yaml_dest_str ); Serial.println( yaml_dest_str ); @@ -352,38 +426,55 @@ void setup() Serial.print( HEAP_AVAILABLE() ); Serial.println(" bytes"); - YAMLParser::setLogLevel( YAML::LogLevelDebug ); // override sketch debug level (otherwise inherited) + YAML::setLogLevel( YAML::LogLevelDebug ); // override sketch debug level (otherwise inherited) + + // test logger, anything under the previously set level shoud be invisible in the console + YAML_LOG_v("This is a verbose message"); + YAML_LOG_d("This is a debug message"); + YAML_LOG_i("This is an info message"); + YAML_LOG_w("This is a warning message"); + YAML_LOG_e("This is an error message"); + + Serial.println("\n"); + YAML_LOG_n("### JSON<=>YAML using libyaml:\n"); - YAML_LOG_n("\n\nJSON=>YAML using libyaml:\n"); + YAML::setJSONIndent(" ", 8 ); // JSON -> two spaces per indent level, unfold objets up to 8 nesting levels + YAML::setYAMLIndent( 3 ); // annoy your friends with 3 spaces indentation + test_Readme_Snippet(); // test basic hello:world snippets from the ReadMe + test_Yaml2JsonPretty(); + test_Yaml2Json(); + test_Json2Yaml(); test_Yaml_String_Parser(); test_Yaml_Stream_Parser(); test_Yaml_String_Loader(); test_Yaml_Stream_Loader(); - YAML_LOG_n("YAMLParser libyaml tests complete"); + YAML_LOG_n("### YAMLParser libyaml tests complete\n"); #if defined HAS_ARDUINOJSON #pragma message "Enabling ArduinoJson tests" - YAML_LOG_n("YAML=>JSON and JSON=>YAML using ArduinoJson\n\n"); + Serial.println("\n"); + YAML_LOG_n("### YAML=>JSON and JSON=>YAML using ArduinoJson\n"); test_deserializeYml_JsonObject_YamlString(); test_deserializeYml_JsonObject_YamlStream(); test_deserializeYml_JsonDocument_YamlStream(); test_deserializeYml_JsonDocument_YamlString(); test_serializeYml_JsonObject_YamlStream(); test_serializeYml_JsonObject_YamlString(); - YAML_LOG_n("ArduinoJson tests complete"); + YAML_LOG_n("### ArduinoJson tests complete\n"); #endif #if defined HAS_CJSON #pragma message "Enabling cJSON tests" - YAML_LOG_n("\n\nYAML=>JSON and JSON=>YAML using cJSON:\n"); + Serial.println("\n"); + YAML_LOG_n("### YAML=>JSON and JSON=>YAML using cJSON:\n"); test_serializeYml_cJson_Stream(); test_serializeYml_cJson_String(); test_deserializeYml_cJson_Stream(); test_deserializeYml_cJson_String(); - YAML_LOG_n("cJSON tests complete"); + YAML_LOG_n("### cJSON tests complete\n"); #endif } diff --git a/library.properties b/library.properties index 1ad8b19..477d725 100644 --- a/library.properties +++ b/library.properties @@ -1,5 +1,5 @@ name=YAMLDuino -version=1.2.7 +version=1.2.8 author=tobozo maintainer=tobozo sentence=A simple and efficient YAML library for embedded C++ diff --git a/src/ArduinoYaml.cpp b/src/ArduinoYaml.cpp index 6c4325d..e4a63b7 100644 --- a/src/ArduinoYaml.cpp +++ b/src/ArduinoYaml.cpp @@ -34,16 +34,81 @@ #define SCALAR_c(x) (const char*)x->data.scalar.value #define SCALAR_s(x) (char*)x->data.scalar.value + +namespace YAML +{ + // defaults + int JSONFoldindDepth = 4; + int YAMLIndentDepth = 2; + String YAML_INDENT_STRING = " "; + String JSON_INDENT_STRING = "\t"; + + #define JSON_INDENT JSON_INDENT_STRING.c_str() + #define YAML_INDENT YAML_INDENT_STRING.c_str() + + void setYAMLIndent( int spaces_per_indent ) + { + if( spaces_per_indent < 2 ) spaces_per_indent = 2; + if( spaces_per_indent > MAX_INDENT_DEPTH ) spaces_per_indent = MAX_INDENT_DEPTH; + YAMLIndentDepth = spaces_per_indent; + YAML_INDENT_STRING = ""; + for( int i=0;idata.scalar.style) { case YAML_PLAIN_SCALAR_STYLE: { - String _scalar = String( (char*)yamlNode->data.scalar.value ); + String _scalar = String( SCALAR_s(yamlNode) ); if( string_has_bool_value(_scalar, value_out) ) { return true; } @@ -103,17 +168,21 @@ void yaml_multiline_escape_string( Stream* stream, const char* str, size_t lengt size_t i; char c; char l = '\0'; - String _str = String(str); + char last_char = str[length-1]; bool has_multiline = _str.indexOf('\n') > -1; - if( has_multiline ) *bytes_out += stream->printf("|\n%s", indent(depth) ); + bool has_ending_lf = last_char == '\n'; + if( has_ending_lf ) { /*_str = _str.substring( 0, -1 );*/ length--; } // remove trailing lf + if( has_multiline ) *bytes_out += stream->printf("|%s\n%s", has_ending_lf?"":"-", indent(depth, YAML::YAML_INDENT) ); + + const char *str_ = _str.c_str(); // modified version of the string for (i = 0; i < length; i++) { - c = *(str + i); + c = *(str_ + i); if( c== '\r' || c=='\n' ) { if( l != '\\' && has_multiline ) { // unescaped \r or \n if(c == '\r') *bytes_out += 1; // ignore \r - else *bytes_out += stream->printf("\n%s", indent(depth) ); // print CRLF + else *bytes_out += stream->printf("\n%s", indent(depth, YAML::YAML_INDENT) ); // print CRLF } else { if(c == '\n') *bytes_out += stream->printf("\\n"); else *bytes_out += stream->printf("\\r"); @@ -147,6 +216,7 @@ void yaml_escape_quoted_string( Stream* stream, const char* str, size_t length, else if (c == '\n') *bytes_out += stream->printf("\\n"); else if (c == '\r') *bytes_out += stream->printf("\\r"); else if (c == '\t') *bytes_out += stream->printf("\\t"); + else if (c == '"') *bytes_out += stream->printf("\\\""); else *bytes_out += stream->printf("%c", c); } } @@ -202,18 +272,6 @@ struct yaml_stream_handler_data_t }; -// struct for recursively parsing yaml_document -struct yaml_traverser_t -{ - yaml_document_t* document; - yaml_node_t* node; - Stream* stream; - JNestingType_t type; - int depth; -}; - - - // stream reader callback int _yaml_stream_reader(void *data, unsigned char *buffer, size_t size, size_t *size_read) { @@ -249,6 +307,112 @@ int _yaml_stream_writer(void *data, unsigned char *buffer, size_t size) +bool scalar_needs_quote( yaml_node_t *node ) +{ + if( node->type != YAML_SCALAR_NODE ) return false; + bool needs_quotes = true; + bool is_bool = false; + bool bool_value = false; + bool is_string = false; + __attribute__((unused)) double number; + char* scalar; + char* end; + scalar = SCALAR_s(node); + number = strtod(scalar, &end); + is_string = (end == scalar || *end); + if( is_string && yaml_node_is_bool( node, &bool_value ) ) { + is_bool = true; + } + if(is_bool) needs_quotes = false; + else if(is_string) needs_quotes = true; + else needs_quotes = false; // number + return needs_quotes; +} + + + +// yaml_document_t traverser (not really a deconstructor) +// output format: JSON +size_t serialize_YamlDocument( yaml_traverser_t *it ) +{ + assert( it ); + assert( it->node ); + assert( it->document ); + assert( it->stream ); + // just some aliasing + auto node = it->node; + auto document = it->document; + auto stream = it->stream; + auto depth = it->depth; + auto nest_type = it->type; + + size_t bytes_out = 0; + int node_count = 0; + + switch (node->type) { + case YAML_SCALAR_NODE: + { + bool needs_quotes = scalar_needs_quote( node ); + if( needs_quotes ) bytes_out += stream->printf("\""); + yaml_escape_quoted_string( stream, SCALAR_c(node), strlen(SCALAR_c(node)), &bytes_out ); + if( needs_quotes ) bytes_out += stream->printf("\""); + } + break; + case YAML_SEQUENCE_NODE: + { + bytes_out += stream->printf("["); + const int node_max = node->data.sequence.items.top - node->data.sequence.items.start; + for (auto item_i = node->data.sequence.items.start; item_i < node->data.sequence.items.top; ++item_i) { + auto node_item = yaml_document_get_node(document, *item_i); + int child_level = node_item->type == YAML_MAPPING_NODE ? depth+1 : depth-1; + yaml_traverser_t seq_item = { document, yaml_document_get_node(document, *item_i), stream, YAMLParser::SEQ_KEY, child_level }; + bytes_out += serialize_YamlDocument( &seq_item ); + node_count++; + if( node_count < node_max ) { + bytes_out += stream->printf(", "); + } + } + bytes_out += stream->printf("]"); + } + break; + case YAML_MAPPING_NODE: + { + bool is_seq = ( depth>0 && nest_type == YAMLParser::SEQ_KEY ); + bool needs_folding = (depth>YAML::JSONFoldindDepth); + bytes_out += stream->printf("{"); + if( !needs_folding ) bytes_out += stream->printf("\n%s", indent(depth+1, YAML::JSON_INDENT) ); + const int node_max = node->data.mapping.pairs.top - node->data.mapping.pairs.start; + for (auto pair_i = node->data.mapping.pairs.start; pair_i < node->data.mapping.pairs.top; ++pair_i) { + auto key = yaml_document_get_node(document, pair_i->key); + auto value = yaml_document_get_node(document, pair_i->value); + if (key->type != YAML_SCALAR_NODE) { + YAML_LOG_e("Mapping key is not scalar (line %lu).", key->start_mark.line); + continue; + } + yaml_traverser_t map_item = { document, value, stream, YAMLParser::MAP_KEY, depth+1 }; + bytes_out += stream->printf("\"%s\": ", SCALAR_c(key) ); + bytes_out += serialize_YamlDocument( &map_item ); + node_count++; + if( node_count < node_max ) { + if( !needs_folding ) bytes_out += stream->printf(",\n%s", indent(depth+1, YAML::JSON_INDENT) ); + else bytes_out += stream->printf(", "); + } + } + if( !needs_folding ) bytes_out += stream->printf("\n%s", indent( is_seq ? depth-1 : depth, YAML::JSON_INDENT) ); + bytes_out += stream->printf("}"); + + } + break; + case YAML_NO_NODE: break; + default: YAML_LOG_e("Unknown node type (line %lu).", node->start_mark.line); break; + } + return bytes_out; +} + + + +// yaml_document_t traverser (not really a deconstructor) +// output format: YAML size_t deserialize_YamlDocument( yaml_traverser_t *it ) { assert( it ); @@ -267,7 +431,7 @@ size_t deserialize_YamlDocument( yaml_traverser_t *it ) switch (node->type) { case YAML_SCALAR_NODE: - if ( nest_type == YAMLParser::SEQ_KEY ) bytes_out += stream->printf("\n%s%s", indent(depth), "- " ); + if ( nest_type == YAMLParser::SEQ_KEY ) bytes_out += stream->printf("\n%s%s", indent(depth, YAML::YAML_INDENT), index() ); yaml_multiline_escape_string( stream, SCALAR_c(node), strlen(SCALAR_c(node)), &bytes_out, depth ); break; case YAML_SEQUENCE_NODE: @@ -284,16 +448,16 @@ size_t deserialize_YamlDocument( yaml_traverser_t *it ) auto key = yaml_document_get_node(document, pair_i->key); auto value = yaml_document_get_node(document, pair_i->value); if (key->type != YAML_SCALAR_NODE) { - YAML_LOG_w("Mapping key is not scalar (line %lu).", key->start_mark.line); + YAML_LOG_e("Mapping key is not scalar (line %lu).", key->start_mark.line); continue; } yaml_traverser_t map_item = { document, value, stream, YAMLParser::MAP_KEY, depth+1 }; - bytes_out += stream->printf("\n%s%s%s: ", is_seqfirst ? indent(parent_level) : indent(depth), is_seqfirst ? "- " : "", SCALAR_c(key) ); + bytes_out += stream->printf("\n%s%s%s: ", is_seqfirst ? indent(parent_level, YAML::YAML_INDENT) : indent(depth, YAML::YAML_INDENT), is_seqfirst ? index() : "", SCALAR_c(key) ); bytes_out += deserialize_YamlDocument( &map_item ); } break; case YAML_NO_NODE: break; - default: YAML_LOG_w("Unknown node type (line %lu).", node->start_mark.line); break; + default: YAML_LOG_e("Unknown node type (line %lu).", node->start_mark.line); break; } return bytes_out; } @@ -301,11 +465,11 @@ size_t deserialize_YamlDocument( yaml_traverser_t *it ) // pure libyaml JSON->YAML stream-to-stream seralization -size_t serializeYml( Stream &stream_in, Stream &stream_out ) +size_t serializeYml( Stream &stream_in, Stream &stream_out, OutputFormat_t output_format ) { YAMLParser* parser = new YAMLParser(); parser->setOutputStream( &stream_out ); - parser->parse( stream_in ); + parser->parse( stream_in, output_format ); auto ret = parser->bytesWritten(); delete parser; return ret; @@ -313,6 +477,17 @@ size_t serializeYml( Stream &stream_in, Stream &stream_out ) +void YAMLParser::setOutputFormat( OutputFormat_t format ) { + output_format = format; + switch(format) { + case OUTPUT_JSON: output_cb = serialize_YamlDocument; YAML::JSONFoldindDepth = -1; break; + case OUTPUT_JSON_PRETTY: output_cb = serialize_YamlDocument; if( YAML::JSONFoldindDepth<0) YAML::JSONFoldindDepth = JSON_FOLDING_DEPTH; break; + default: + case OUTPUT_YAML: output_cb = deserialize_YamlDocument; break; + } +} + + YAMLParser::YAMLParser() { _yaml_string = ""; @@ -337,22 +512,23 @@ void YAMLParser::setLogLevel( YAML::LogLevel_t level ) } -void YAMLParser::parse() +void YAMLParser::parse( OutputFormat_t format ) { yaml_node_t* node; if (node = yaml_document_get_root_node(&document), !node) { - YAML_LOG_w("No document defined."); + YAML_LOG_e("No document defined."); return; } yaml_traverser_t doc = { &document, node, _yaml_stream, YAMLParser::NONE, 0 }; - size_t bytes_out = deserialize_YamlDocument( &doc ); + setOutputFormat( format ); + size_t bytes_out = output_cb( &doc ); YAML_LOG_d("written %d bytes", bytes_out ); } -void YAMLParser::parse( Stream &yaml_or_json_stream ) +void YAMLParser::parse( Stream &yaml_or_json_stream, OutputFormat_t format ) { if (!yaml_parser_initialize(&parser)) { YAML_LOG_e("Failed to initialize parser!\n", stderr); @@ -369,19 +545,20 @@ void YAMLParser::parse( Stream &yaml_or_json_stream ) yaml_node_t* node; if (node = yaml_document_get_root_node(&document), !node) { - YAML_LOG_w("No document defined."); + YAML_LOG_e("No document defined."); return; } yaml_traverser_t doc = { &document, node, _yaml_stream, YAMLParser::NONE, 0 }; - size_t bytes_out = deserialize_YamlDocument( &doc ); + setOutputFormat( format ); + size_t bytes_out = output_cb( &doc ); YAML_LOG_d("written %d bytes", bytes_out ); } -void YAMLParser::parse( const char* yaml_or_json_str ) +void YAMLParser::parse( const char* yaml_or_json_str, OutputFormat_t format ) { assert( yaml_or_json_str ); @@ -399,12 +576,13 @@ void YAMLParser::parse( const char* yaml_or_json_str ) yaml_node_t* node; if (node = yaml_document_get_root_node(&document), !node) { - YAML_LOG_w("No document defined."); + YAML_LOG_e("No document defined."); return; } yaml_traverser_t doc = { &document, node, _yaml_stream, YAMLParser::NONE, 0 }; - size_t bytes_out = deserialize_YamlDocument( &doc ); + setOutputFormat( format ); + size_t bytes_out = output_cb( &doc ); YAML_LOG_d("written %d bytes", bytes_out ); } @@ -424,28 +602,33 @@ void YAMLParser::load( Stream &yaml_or_json_stream ) yaml_stream_handler_data_t shd = { &yaml_or_json_stream, &_bytes_read }; yaml_parser_set_input(&parser, &_yaml_stream_reader, &shd); - loadDocument(); + + _loadDocument(); } void YAMLParser::load( const char* yaml_or_json_str ) { assert( yaml_or_json_str ); + _yaml_string = ""; // reset internal output stream _bytes_read = strlen(yaml_or_json_str); // length is already known _bytes_written = 0; // reset yaml output length + if ( !yaml_parser_initialize(&parser) ) { handle_parser_error(&parser); YAML_LOG_e("[FATAL] could not initialize parser"); return; } + yaml_parser_set_input_string(&parser, (const unsigned char*)yaml_or_json_str, _bytes_read ); - loadDocument(); + + _loadDocument(); } -// called by load(const char*) and load(Stream&) -void YAMLParser::loadDocument() +// private, called by load(const char*) and load(Stream&) +void YAMLParser::_loadDocument() { yaml_document_t _tmpdoc; yaml_emitter_t emitter; @@ -500,10 +683,10 @@ void YAMLParser::handle_parser_error(yaml_parser_t *p) void YAMLParser::handle_emitter_error(yaml_emitter_t *e) { switch (e->error) { - case YAML_MEMORY_ERROR: YAML_LOG_e("[MEMORY ERROR]: Not enough memory for emitting"); break; - case YAML_WRITER_ERROR: YAML_LOG_e("[WRITER ERROR]: %s", e->problem); break; - case YAML_EMITTER_ERROR: YAML_LOG_e("[EMITTER ERROR]: %s", e->problem); break; - default: /* Couldn't happen. */ YAML_LOG_e( "[INTERNAL ERROR]"); break; + case YAML_MEMORY_ERROR: YAML_LOG_e("[MEM]: Not enough memory for emitting"); break; + case YAML_WRITER_ERROR: YAML_LOG_e("[WRI]: %s", e->problem); break; + case YAML_EMITTER_ERROR: YAML_LOG_e("[EMI]: %s", e->problem); break; + default:/*Couldn't happen*/YAML_LOG_e("[INT]"); break; } } @@ -521,7 +704,7 @@ void YAMLParser::handle_emitter_error(yaml_emitter_t *e) double number; char* scalar; char* end; - scalar = (char *)yamlNode->data.scalar.value; + scalar = SCALAR_s(yamlNode); number = strtod(scalar, &end); bool is_bool = false; bool bool_value = false; @@ -536,14 +719,12 @@ void YAMLParser::handle_emitter_error(yaml_emitter_t *e) if(is_bool) array.add( bool_value ); else if(is_string) array.add( scalar ); else array.add( number ); - //YAML_LOG_d("[SEQ][%s][%d] => %s(%s)", nodename, array.size()-1, is_string?"string":"number", is_string?scalar:String(number).c_str() ); } break; case YAMLParser::MAP_KEY: if(is_bool) jsonNode[nodename] = bool_value; else if(is_string) jsonNode[nodename] = scalar; else jsonNode[nodename] = number; - //YAML_LOG_d("[MAP][%d][%s] => %s(%s)", jsonNode.size()-1, nodename, is_string?"string":"number", is_string?scalar:String(number).c_str() ); break; default: YAML_LOG_e("Error invalid nesting type"); break; } @@ -569,7 +750,6 @@ void YAMLParser::handle_emitter_error(yaml_emitter_t *e) deserializeYml_JsonObject( document, itemNode, jsonNode, YAMLParser::SEQ_KEY, _nodeItemName.c_str(), depth+1 ); } } - //YAML_LOG_d("[ARR][%s] has %d items", nodename, tmpArray.size() ); } break; case YAML_MAPPING_NODE: @@ -582,16 +762,16 @@ void YAMLParser::handle_emitter_error(yaml_emitter_t *e) key = yaml_document_get_node(document, pair_i->key); value = yaml_document_get_node(document, pair_i->value); if (key->type != YAML_SCALAR_NODE) { - YAML_LOG_w("Mapping key is not scalar (line %lu, val=%s).", key->start_mark.line, (const char*)value->data.scalar.value); + YAML_LOG_e("Mapping key is not scalar (line %lu, val=%s).", key->start_mark.line, SCALAR_c(value) ); continue; } - tmpNode.createNestedObject((char*)key->data.scalar.value); - deserializeYml_JsonObject( document, value, tmpNode, YAMLParser::MAP_KEY, (const char*)key->data.scalar.value, depth+1 ); + tmpNode.createNestedObject( SCALAR_s(key) ); + deserializeYml_JsonObject( document, value, tmpNode, YAMLParser::MAP_KEY, SCALAR_c(key), depth+1 ); } } break; case YAML_NO_NODE: YAML_LOG_e("YAML_NO_NODE"); break; - default: YAML_LOG_w("Unknown node type (line %lu).", yamlNode->start_mark.line); break; + default: YAML_LOG_e("Unknown node type (line %lu).", yamlNode->start_mark.line); break; } } @@ -612,11 +792,11 @@ void YAMLParser::handle_emitter_error(yaml_emitter_t *e) int i = 0; for (JsonPair pair : object) { bool is_seqfirst = (i++==0 && nt==YAMLParser::SEQ_KEY); - out_size += out.printf("\n%s%s%s: ", is_seqfirst ? indent(parent_level) : indent(depth), is_seqfirst ? "- " : "", pair.key().c_str() ); + out_size += out.printf("\n%s%s%s: ", is_seqfirst ? indent(parent_level, YAML::YAML_INDENT) : indent(depth, YAML::YAML_INDENT), is_seqfirst ? index() : "", pair.key().c_str() ); out_size += serializeYml_JsonVariant( pair.value(), out, depth+1, YAMLParser::MAP_KEY ); } } else if( !root.isNull() ) { - if( nt == YAMLParser::SEQ_KEY ) out_size += out.printf("\n%s%s", indent(depth), "- " ); + if( nt == YAMLParser::SEQ_KEY ) out_size += out.printf("\n%s%s", indent(depth, YAML::YAML_INDENT), index() ); yaml_multiline_escape_string(&out, root.as().c_str(), root.as().length(), &out_size, depth); } else { YAML_LOG_e("Error, root is null"); @@ -689,7 +869,7 @@ void YAMLParser::handle_emitter_error(yaml_emitter_t *e) double number; char * scalar; char * end; - scalar = (char *)yamlNode->data.scalar.value; + scalar = SCALAR_s(yamlNode); number = strtod(scalar, &end); if( (end == scalar || *end) ) { // string or bool bool bool_value; @@ -720,15 +900,15 @@ void YAMLParser::handle_emitter_error(yaml_emitter_t *e) key = yaml_document_get_node(document, pair_i->key); value = yaml_document_get_node(document, pair_i->value); if (key->type != YAML_SCALAR_NODE) { - YAML_LOG_w("Mapping key is not scalar (line %lu).", key->start_mark.line); + YAML_LOG_e("Mapping key is not scalar (line %lu).", key->start_mark.line); continue; } - cJSON_AddItemToObject(object, (const char *)key->data.scalar.value, deserializeYml_cJSONObject(document, value)); + cJSON_AddItemToObject(object, SCALAR_c(key), deserializeYml_cJSONObject(document, value)); } } break; default: - YAML_LOG_w("Unknown node type (line %lu).", yamlNode->start_mark.line); + YAML_LOG_e("Unknown node type (line %lu).", yamlNode->start_mark.line); object = NULL; } return object; @@ -755,7 +935,7 @@ void YAMLParser::handle_emitter_error(yaml_emitter_t *e) while (current_item) { bool is_seqfirst = (i++==0 && nt==YAMLParser::SEQ_KEY); char* key = current_item->string; - out_size += out.printf("\n%s%s%s: ", is_seqfirst ? indent(parent_level) : indent(depth), is_seqfirst ? "- " : "", key ); + out_size += out.printf("\n%s%s%s: ", is_seqfirst ? indent(parent_level, YAML::YAML_INDENT) : indent(depth, YAML::YAML_INDENT), is_seqfirst ? index() : "", key ); out_size += serializeYml_cJSONObject( current_item, out, depth+1, YAMLParser::MAP_KEY ); current_item = current_item->next; } @@ -771,7 +951,7 @@ void YAMLParser::handle_emitter_error(yaml_emitter_t *e) } } size_t value_len = strlen(value); - if( nt == YAMLParser::SEQ_KEY ) out_size += out.printf("\n%s%s", indent(depth), "- " ); + if( nt == YAMLParser::SEQ_KEY ) out_size += out.printf("\n%s%s", indent(depth, YAML::YAML_INDENT), index() ); yaml_multiline_escape_string(&out, value, value_len, &out_size, depth); if( value_needs_free ) cJSON_free( value ); } diff --git a/src/ArduinoYaml.hpp b/src/ArduinoYaml.hpp index 4a727db..de5bb08 100644 --- a/src/ArduinoYaml.hpp +++ b/src/ArduinoYaml.hpp @@ -30,10 +30,6 @@ */ -// fuck you espressif, this warning is treaded as error -//#pragma GCC diagnostic ignored "-Wunused-variable" -//#pragma GCC diagnostic ignored "-Wunused-but-set-variable" - #include "logger.hpp" extern "C" { @@ -47,7 +43,8 @@ extern "C" { #if !defined YAML_DISABLE_ARDUINOJSON #if defined ARDUINO_ARCH_SAMD || defined ARDUINO_ARCH_RP2040 || defined ESP8266 - // those platforms don't have built-in cJSON so assume ArduinoJson is in use + // those platforms don't have built-in cJSON and __has_include() macro is limited to + // the sketch folder, so assume ArduinoJson is in use #include #include #include @@ -61,24 +58,36 @@ extern "C" { #endif +#define YAML_SCALAR_SPACE " " // YAML is indented with spaces (2 or more), not tabs +#define JSON_SCALAR_TAB "\t" // JSON is indented with one tab as a default, this can be changed later +#define JSON_FOLDING_DEPTH 4 // lame fact: folds on objects, not on arrays +#define MAX_INDENT_DEPTH 16 // max amount of space/tab per indent level, doesn't seem reasonable to do more on embedded projects + + +namespace YAML +{ + void setYAMLIndent( int spaces_per_indent=2 ); // min=2, max=16 + void setJSONIndent( const char* spaces_or_tabs=JSON_SCALAR_TAB, int folding_depth=JSON_FOLDING_DEPTH ); +}; // provide a default String::Stream reader/writer for internals class StringStream : public Stream { public: StringStream(String &s) : str(s), pos(0) {} - virtual ~StringStream() { }; + virtual ~StringStream() {}; virtual int available() { return str.length() - pos; } virtual int read() { return pos < str.length() ? str[pos++] : -1; } virtual int peek() { return pos < str.length() ? str[pos] : -1; } - virtual void flush() {} virtual size_t write(uint8_t c) { str += (char)c; return 1; } + virtual void flush() {} private: String &str; unsigned int pos; }; + // the base class class YAMLParser { @@ -86,39 +95,73 @@ class YAMLParser YAMLParser(); ~YAMLParser(); + // generic node type shared between json and yaml iterators + enum JNestingType_t { NONE, SEQ_KEY, MAP_KEY }; + // wrapper struct passed when recursively parsing yaml_document + struct yaml_traverser_t + { + yaml_document_t* document; + yaml_node_t* node; + Stream* stream; + JNestingType_t type; + int depth; + }; + // output emitter signature, points to serialize or deserialize (defaults to YAML output) + typedef size_t (*yaml_doc_processor_cb)( yaml_traverser_t *it ); + // available output formats + enum OutputFormat_t { OUTPUT_YAML, OUTPUT_JSON, OUTPUT_JSON_PRETTY }; + + // output controls + void setOutputFormat( OutputFormat_t format ); void setOutputStream( Stream* stream ) { _yaml_stream = stream; } + // yaml/json loaders (populates this.document) void load( const char* yaml_or_json_str ); void load( Stream &yaml_or_json_stream ); - void parse(); - void parse( const char* yaml_or_json_str ); - void parse( Stream &yaml_or_json_stream ); + // the swiss-army knife functions: + // parse any of yaml/json input, and output as yaml, json ugly or json pretty + void parse( OutputFormat_t format=OUTPUT_YAML ); + void parse( const char* yaml_or_json_str, OutputFormat_t format=OUTPUT_YAML ); + void parse( Stream &yaml_or_json_stream, OutputFormat_t format=OUTPUT_YAML ); + + // explicit JSON exporters + template + void toJson( T &yaml, bool pretty = true ) { load( yaml ); toJson( pretty ); } + void toJson( bool pretty = true ) { parse( pretty ? OUTPUT_JSON_PRETTY : OUTPUT_JSON ); } + // various getters yaml_document_t* getDocument() { return &document; } String getYamlString() { return _yaml_string; } size_t bytesWritten() { return _bytes_written; } size_t bytesRead() { return _bytes_read; } - enum JNestingType_t { NONE, SEQ_KEY, MAP_KEY }; + // error handling static void handle_parser_error(yaml_parser_t *parser); static void handle_emitter_error(yaml_emitter_t* emitter); static void setLogLevel( YAML::LogLevel_t level ); + private: size_t _bytes_read; size_t _bytes_written; String _yaml_string; Stream *_yaml_stream = nullptr; StringStream *_yaml_string_stream_ptr = nullptr; - void loadDocument(); + void _loadDocument(); yaml_document_t document; - yaml_parser_t parser; + yaml_parser_t parser; + OutputFormat_t output_format; + yaml_doc_processor_cb output_cb; }; typedef YAMLParser::JNestingType_t JNestingType_t; +typedef YAMLParser::yaml_traverser_t yaml_traverser_t; +typedef YAMLParser::OutputFormat_t OutputFormat_t; + + +// Pure libyaml JSON->YAML stream-to-stream seralization +size_t serializeYml( Stream &json_src_stream, Stream &yml_dest_stream, OutputFormat_t format=YAMLParser::OUTPUT_YAML ); -// JSON stream to JsonObject to YAML stream -size_t serializeYml( Stream &json_src_stream, Stream &yml_dest_stream ); #if defined HAS_ARDUINOJSON diff --git a/src/logger.hpp b/src/logger.hpp index fd53f6f..2600749 100644 --- a/src/logger.hpp +++ b/src/logger.hpp @@ -47,6 +47,10 @@ #define HEAP_AVAILABLE() ESP.getFreeHeap() #define YAML_DEFAULT_LOG_LEVEL (LogLevel_t)ARDUHAL_LOG_LEVEL #define YAML_PATHNAME pathToFileName + // uncomment this if your espresssif32 package complains + //#pragma GCC diagnostic ignored "-Wunused-variable" + //#pragma GCC diagnostic ignored "-Wunused-but-set-variable" + #elif defined ESP8266 #include "Esp.h" // bring esp8266-arduino specifics to scope #define HEAP_AVAILABLE() ESP.getFreeHeap()