json start/end position implementation (#4517)

* Add implementation to retrieve start and end positions of json during parse

* Add more unit tests and add start/stop parsing for arrays

* Add raw value for all types

* Add more tests and fix compiler warning

* Amalgamate

* Fix CLang GCC warnings

* Fix error in build

* Style using astyle 3.1

* Fix whitespace changes

* revert

* more whitespace reverts

* Address PR comments

* Fix failing issues

* More whitespace reverts

* Address remaining PR comments

* Address comments

* Switch to using custom base class instead of default basic_json

* Adding a basic using for a json using the new base class. Also address PR comments and fix CI failures

* Address decltype comments

* Diagnostic positions macro (#4)

Co-authored-by: Sush Shringarputale <sushring@linux.microsoft.com>

* Fix missed include deletion

* Add docs and address other PR comments (#5)

* Add docs and address other PR comments

---------

Co-authored-by: Sush Shringarputale <sushring@linux.microsoft.com>

* Address new PR comments and fix CI tests for documentation

* Update documentation based on feedback (#6)

---------

Co-authored-by: Sush Shringarputale <sushring@linux.microsoft.com>

* Address std::size_t and other comments

* Fix new CI issues

* Fix lcov

* Improve lcov case with update to handle_diagnostic_positions call for discarded values

* Fix indentation of LCOV_EXCL_STOP comments

* fix amalgamation astyle issue

---------

Co-authored-by: Sush Shringarputale <sushring@linux.microsoft.com>

CMakeLists.txt

@@ -40,6 +40,7 @@ endif()
 option(JSON_BuildTests                     "Build the unit tests when BUILD_TESTING is enabled." ${JSON_BuildTests_INIT})
 option(JSON_CI                             "Enable CI build targets." OFF)
 option(JSON_Diagnostics                    "Use extended diagnostic messages." OFF)
+option(JSON_Diagnostic_Positions           "Enable diagnostic positions." OFF)
 option(JSON_GlobalUDLs                     "Place user-defined string literals in the global namespace." ON)
 option(JSON_ImplicitConversions            "Enable implicit conversions." ON)
 option(JSON_DisableEnumSerialization       "Disable default integer enum serialization." OFF)
@@ -96,6 +97,10 @@ if (JSON_Diagnostics)
     message(STATUS "Diagnostics enabled (JSON_DIAGNOSTICS=1)")
 endif()
 
+if (JSON_Diagnostic_Positions)
+    message(STATUS "Diagnostic positions enabled")
+endif()
+
 if (NOT JSON_GlobalUDLs)
     message(STATUS "User-defined string literals are not put in the global namespace (JSON_USE_GLOBAL_UDLS=0)")
 endif()
@@ -123,6 +128,7 @@ target_compile_definitions(
     $<$<NOT:$<BOOL:${JSON_ImplicitConversions}>>:JSON_USE_IMPLICIT_CONVERSIONS=0>
     $<$<BOOL:${JSON_DisableEnumSerialization}>:JSON_DISABLE_ENUM_SERIALIZATION=1>
     $<$<BOOL:${JSON_Diagnostics}>:JSON_DIAGNOSTICS=1>
+    $<$<BOOL:${JSON_Diagnostic_Positions}>:JSON_DIAGNOSTIC_POSITIONS=1>
     $<$<BOOL:${JSON_LegacyDiscardedValueComparison}>:JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON=1>
 )
 

docs/examples/diagnostic_positions.cpp

@@ -0,0 +1,51 @@
+#include <iostream>
+
+#define JSON_DIAGNOSTIC_POSITIONS 1
+#include <nlohmann/json.hpp>
+
+using json = nlohmann::json;
+
+int main()
+{
+    std::string json_string = R"(
+    {
+        "address": {
+            "street": "Fake Street",
+            "housenumber": 1
+        }
+    }
+    )";
+    json j = json::parse(json_string);
+
+    std::cout << "Root diagnostic positions: \n";
+    std::cout << "\tstart_pos: " << j.start_pos() << '\n';
+    std::cout << "\tend_pos:" << j.end_pos() << "\n";
+    std::cout << "Original string: \n";
+    std::cout << "{\n        \"address\": {\n            \"street\": \"Fake Street\",\n            \"housenumber\": 1\n        }\n    }" << "\n";
+    std::cout << "Parsed string: \n";
+    std::cout << json_string.substr(j.start_pos(), j.end_pos() - j.start_pos()) << "\n\n";
+
+    std::cout << "address diagnostic positions: \n";
+    std::cout << "\tstart_pos:" << j["address"].start_pos() << '\n';
+    std::cout << "\tend_pos:" << j["address"].end_pos() << "\n\n";
+    std::cout << "Original string: \n";
+    std::cout << "{            \"street\": \"Fake Street\",\n            \"housenumber\": 1\n        }" << "\n";
+    std::cout << "Parsed string: \n";
+    std::cout << json_string.substr(j["address"].start_pos(), j["address"].end_pos() - j["address"].start_pos()) << "\n\n";
+
+    std::cout << "street diagnostic positions: \n";
+    std::cout << "\tstart_pos:" << j["address"]["street"].start_pos() << '\n';
+    std::cout << "\tend_pos:" << j["address"]["street"].end_pos() << "\n\n";
+    std::cout << "Original string: \n";
+    std::cout << "\"Fake Street\"" << "\n";
+    std::cout << "Parsed string: \n";
+    std::cout << json_string.substr(j["address"]["street"].start_pos(), j["address"]["street"].end_pos() - j["address"]["street"].start_pos()) << "\n\n";
+
+    std::cout << "housenumber diagnostic positions: \n";
+    std::cout << "\tstart_pos:" << j["address"]["housenumber"].start_pos() << '\n';
+    std::cout << "\tend_pos:" << j["address"]["housenumber"].end_pos() << "\n\n";
+    std::cout << "Original string: \n";
+    std::cout << "1" << "\n";
+    std::cout << "Parsed string: \n";
+    std::cout << json_string.substr(j["address"]["housenumber"].start_pos(), j["address"]["housenumber"].end_pos() - j["address"]["housenumber"].start_pos()) << "\n\n";
+}

docs/examples/diagnostic_positions.output

@@ -0,0 +1,50 @@
+Root diagnostic positions: 
+	start_pos: 5
+	end_pos:109
+Original string: 
+{
+        "address": {
+            "street": "Fake Street",
+            "housenumber": 1
+        }
+    }
+Parsed string: 
+{
+        "address": {
+            "street": "Fake Street",
+            "housenumber": 1
+        }
+    }
+
+address diagnostic positions: 
+	start_pos:26
+	end_pos:103
+
+Original string: 
+{            "street": "Fake Street",
+            "housenumber": 1
+        }
+Parsed string: 
+{
+            "street": "Fake Street",
+            "housenumber": 1
+        }
+
+street diagnostic positions: 
+	start_pos:50
+	end_pos:63
+
+Original string: 
+"Fake Street"
+Parsed string: 
+"Fake Street"
+
+housenumber diagnostic positions: 
+	start_pos:92
+	end_pos:93
+
+Original string: 
+1
+Parsed string: 
+1
+

docs/mkdocs/docs/api/macros/json_diagnostic_positions.md

@@ -0,0 +1,61 @@
+# JSON_DIAGNOSTIC_POSITIONS
+
+```cpp
+#define JSON_DIAGNOSTIC_POSITIONS /* value */
+```
+
+This macro enables position diagnostics for generated JSON objects.
+
+When enabled, two new properties: `start_pos()` and `end_pos()` are added to `nlohmann::basic_json` objects and fields. `start_pos()` returns the start 
+position of that JSON object/field in the original string the object was parsed from. Likewise, `end_pos()` returns the end position of that JSON
+object/field in the original string the object was parsed from.
+
+`start_pos()` returns the first character of a given element in the original JSON string, while `end_pos()` returns the character following the last
+character. For objects and arrays, the first and last characters correspond to the opening or closing braces/brackets, respectively. For fields, the first
+and last character represent the opening and closing quotes or the first and last character of the field's numerical or predefined value
+(true/false/null), respectively.
+
+Given the above, `end_pos() - start_pos()` for an object or field provides the length of the string representation for that object or field, including the
+opening or closing braces, brackets, or quotes.
+
+`start_pos()` and `end_pos()` are only set if the JSON object was parsed using `parse()`. For all other cases, `std::string::npos` will be returned.
+
+Note that enabling this macro increases the size of every JSON value by two `std::size_t` fields and adds
+slight runtime overhead.
+
+## Default definition
+
+The default value is `0` (position diagnostics are switched off).
+
+```cpp
+#define JSON_DIAGNOSTIC_POSITIONS 0
+```
+
+When the macro is not defined, the library will define it to its default value.
+
+## Notes
+
+!!! hint "CMake option"
+
+    Diagnostic messages can also be controlled with the CMake option
+    [`JSON_Diagnostic_Positions`](../../integration/cmake.md#json_diagnostic_positions) (`OFF` by default)
+    which defines `JSON_DIAGNOSTIC_POSITIONS` accordingly.
+
+## Examples
+
+??? example "Example 1: retrieving positions"
+
+    ```cpp
+    --8<-- "examples/diagnostic_positions.cpp"
+    ```
+
+    Output:
+
+    ```
+    --8<-- "examples/diagnostic_positions.output"
+    ```
+
+    The output shows the start/end positions of all the objects and fields in the JSON string.
+
+## Version history
+

docs/mkdocs/docs/integration/cmake.md

@@ -135,6 +135,9 @@ Enable CI build targets. The exact targets are used during the several CI steps
 
 Enable [extended diagnostic messages](../home/exceptions.md#extended-diagnostic-messages) by defining macro [`JSON_DIAGNOSTICS`](../api/macros/json_diagnostics.md). This option is `OFF` by default.
 
+### `JSON_Diagnostic_Positions`
+Enable position diagnostics by defining macro [`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md). This option is off by default.
+
 ### `JSON_DisableEnumSerialization`
 
 Disable default `enum` serialization by defining the macro

include/nlohmann/detail/abi_macros.hpp

@@ -26,6 +26,10 @@
     #define JSON_DIAGNOSTICS 0
 #endif
 
+#ifndef JSON_DIAGNOSTIC_POSITIONS
+    #define JSON_DIAGNOSTIC_POSITIONS 0
+#endif
+
 #ifndef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON
     #define JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 0
 #endif
@@ -36,6 +40,12 @@
     #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS
 #endif
 
+#if JSON_DIAGNOSTIC_POSITIONS
+    #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS _dp
+#else
+    #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS
+#endif
+
 #if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON
     #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON _ldvcmp
 #else
@@ -47,14 +57,15 @@
 #endif
 
 // Construct the namespace ABI tags component
-#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b) json_abi ## a ## b
-#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b) \
-    NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b)
+#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c) json_abi ## a ## b ## c
+#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c) \
+    NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c)
 
 #define NLOHMANN_JSON_ABI_TAGS                                       \
     NLOHMANN_JSON_ABI_TAGS_CONCAT(                                   \
             NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS,                       \
-            NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON)
+            NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON, \
+            NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS)
 
 // Construct the namespace version component
 #define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \

include/nlohmann/detail/input/binary_reader.hpp

@@ -65,7 +65,7 @@ static inline bool little_endianness(int num = 1) noexcept
 /*!
 @brief deserialization of CBOR, MessagePack, and UBJSON values
 */
-template<typename BasicJsonType, typename InputAdapterType, typename SAX = json_sax_dom_parser<BasicJsonType>>
+template<typename BasicJsonType, typename InputAdapterType, typename SAX = json_sax_dom_parser<BasicJsonType, InputAdapterType>>
 class binary_reader
 {
     using number_integer_t = typename BasicJsonType::number_integer_t;

include/nlohmann/detail/input/input_adapters.hpp

@@ -462,6 +462,9 @@ typename container_input_adapter_factory_impl::container_input_adapter_factory<C
     return container_input_adapter_factory_impl::container_input_adapter_factory<ContainerType>::create(container);
 }
 
+// specialization for std::string
+using string_input_adapter_type = decltype(input_adapter(std::declval<std::string>()));
+
 #ifndef JSON_NO_IO
 // Special cases with fast paths
 inline file_input_adapter input_adapter(std::FILE* file)