Emitted newlines no longer confuse comment merge, closes #344, closes #351, closes #354

Thanks for these test cases. The common issue was that occasionally an emitted piece of Cpp1 contained a newline character, which was not accounted for in the code that did comment merging.

Thanks also for the PRs for proposed fixes. I looked at those too and tried them out. In the end, I found that what worked best and most generally was to break apart strings that contain newlines into one chunk per line, and this allowed the comment merging to happen correctly by avoiding the hidden-to-the-comment-merger line change.

So this commit is going with that solution rather than the other proposed PRs, but the other PRs helped find this root cause and shed light on a general solution, as did the test cases. Thanks again for both!

Author: hsutter

regression-tests/test-results/clang-12/mixed-float-literals.cpp.output

@@ -1,121 +1,121 @@
-mixed-float-literals.cpp:13:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:14:3: warning: expression result unused [-Wunused-value]
   123;
   ^~~
-mixed-float-literals.cpp:14:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:15:3: warning: expression result unused [-Wunused-value]
   123u;
   ^~~~
-mixed-float-literals.cpp:15:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:16:3: warning: expression result unused [-Wunused-value]
   123ul;
   ^~~~~
-mixed-float-literals.cpp:16:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:17:3: warning: expression result unused [-Wunused-value]
   123ull;
   ^~~~~~
-mixed-float-literals.cpp:17:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:18:3: warning: expression result unused [-Wunused-value]
   123l;
   ^~~~
-mixed-float-literals.cpp:18:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:19:3: warning: expression result unused [-Wunused-value]
   123ll;
   ^~~~~
-mixed-float-literals.cpp:19:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:20:3: warning: expression result unused [-Wunused-value]
   123'456ll;
   ^~~~~~~~~
-mixed-float-literals.cpp:20:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:21:3: warning: expression result unused [-Wunused-value]
   123;
   ^~~
-mixed-float-literals.cpp:21:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:22:3: warning: expression result unused [-Wunused-value]
   123U;
   ^~~~
-mixed-float-literals.cpp:22:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:23:3: warning: expression result unused [-Wunused-value]
   123UL;
   ^~~~~
-mixed-float-literals.cpp:23:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:24:3: warning: expression result unused [-Wunused-value]
   123ULL;
   ^~~~~~
-mixed-float-literals.cpp:24:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:25:3: warning: expression result unused [-Wunused-value]
   123L;
   ^~~~
-mixed-float-literals.cpp:25:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:26:3: warning: expression result unused [-Wunused-value]
   123LL;
   ^~~~~
-mixed-float-literals.cpp:26:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:27:3: warning: expression result unused [-Wunused-value]
   123'456LL;
   ^~~~~~~~~
-mixed-float-literals.cpp:28:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:29:3: warning: expression result unused [-Wunused-value]
   123'456.0f;
   ^~~~~~~~~~
-mixed-float-literals.cpp:29:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:30:3: warning: expression result unused [-Wunused-value]
   123'456.f;
   ^~~~~~~~~
-mixed-float-literals.cpp:30:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:31:3: warning: expression result unused [-Wunused-value]
   456.0;
   ^~~~~
-mixed-float-literals.cpp:31:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:32:3: warning: expression result unused [-Wunused-value]
   456.;
   ^~~~
-mixed-float-literals.cpp:32:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:33:3: warning: expression result unused [-Wunused-value]
   1.0e10;
   ^~~~~~
-mixed-float-literals.cpp:33:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:34:3: warning: expression result unused [-Wunused-value]
   1.0e+10;
   ^~~~~~~
-mixed-float-literals.cpp:34:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:35:3: warning: expression result unused [-Wunused-value]
   1.0e-10;
   ^~~~~~~
-mixed-float-literals.cpp:35:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:36:3: warning: expression result unused [-Wunused-value]
   1.0e-10f;
   ^~~~~~~~
-mixed-float-literals.cpp:36:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:37:3: warning: expression result unused [-Wunused-value]
   1.e-10;
   ^~~~~~
-mixed-float-literals.cpp:37:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:38:3: warning: expression result unused [-Wunused-value]
   1.e-10f;
   ^~~~~~~
-mixed-float-literals.cpp:38:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:39:3: warning: expression result unused [-Wunused-value]
   1e-10;
   ^~~~~
-mixed-float-literals.cpp:39:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:40:3: warning: expression result unused [-Wunused-value]
   1e-10f;
   ^~~~~~
-mixed-float-literals.cpp:40:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:41:3: warning: expression result unused [-Wunused-value]
   1e-1'0;
   ^~~~~~
-mixed-float-literals.cpp:41:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:42:3: warning: expression result unused [-Wunused-value]
   123'456.0F;
   ^~~~~~~~~~
-mixed-float-literals.cpp:42:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:43:3: warning: expression result unused [-Wunused-value]
   123'456.F;
   ^~~~~~~~~
-mixed-float-literals.cpp:43:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:44:3: warning: expression result unused [-Wunused-value]
   456.0;
   ^~~~~
-mixed-float-literals.cpp:44:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:45:3: warning: expression result unused [-Wunused-value]
   456.;
   ^~~~
-mixed-float-literals.cpp:45:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:46:3: warning: expression result unused [-Wunused-value]
   1.0E10;
   ^~~~~~
-mixed-float-literals.cpp:46:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:47:3: warning: expression result unused [-Wunused-value]
   1.0E+10;
   ^~~~~~~
-mixed-float-literals.cpp:47:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:48:3: warning: expression result unused [-Wunused-value]
   1.0E-10;
   ^~~~~~~
-mixed-float-literals.cpp:48:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:49:3: warning: expression result unused [-Wunused-value]
   1.0E-10F;
   ^~~~~~~~
-mixed-float-literals.cpp:49:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:50:3: warning: expression result unused [-Wunused-value]
   1.E-10;
   ^~~~~~
-mixed-float-literals.cpp:50:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:51:3: warning: expression result unused [-Wunused-value]
   1.E-10F;
   ^~~~~~~
-mixed-float-literals.cpp:51:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:52:3: warning: expression result unused [-Wunused-value]
   1E-10;
   ^~~~~
-mixed-float-literals.cpp:52:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:53:3: warning: expression result unused [-Wunused-value]
   1E-10F;
   ^~~~~~
-mixed-float-literals.cpp:53:3: warning: expression result unused [-Wunused-value]
+mixed-float-literals.cpp:54:3: warning: expression result unused [-Wunused-value]
   1E-1'0;
   ^~~~~~
 mixed-float-literals.cpp2:59:3: warning: expression result unused [-Wunused-value]

regression-tests/test-results/mixed-as-for-variant-20-types.cpp

@@ -6,6 +6,7 @@
 #include "cpp2util.h"
 
 
+
 //=== Cpp2 type definitions and function declarations ===========================
 
 template<int I>
@@ -14,6 +15,7 @@ struct X { operator int() const { return I; } };
 #line 4 "mixed-as-for-variant-20-types.cpp2"
 [[nodiscard]] auto main() -> int;
 
+
 //=== Cpp2 function definitions =================================================
 
 

regression-tests/test-results/mixed-bounds-check.cpp

@@ -6,13 +6,15 @@
 #include "cpp2util.h"
 
 
+
 //=== Cpp2 type definitions and function declarations ===========================
 
 
 #include <vector>
 
 #line 4 "mixed-bounds-check.cpp2"
 [[nodiscard]] auto main() -> int;
+    
 
 //=== Cpp2 function definitions =================================================
 

regression-tests/test-results/mixed-bounds-safety-with-assert-2.cpp

@@ -6,11 +6,13 @@
 #include "cpp2util.h"
 
 
+
 //=== Cpp2 type definitions and function declarations ===========================
 
 
 #line 2 "mixed-bounds-safety-with-assert-2.cpp2"
 [[nodiscard]] auto main() -> int;
+    
 
 #line 10 "mixed-bounds-safety-with-assert-2.cpp2"
 auto add_42_to_subrange(auto& rng, cpp2::in<int> start, cpp2::in<int> end) -> void;
@@ -20,6 +22,7 @@ auto add_42_to_subrange(auto& rng, cpp2::in<int> start, cpp2::in<int> end) -> vo
 #include <span>
 #include <iostream>
 
+
 //=== Cpp2 function definitions =================================================
 
 

regression-tests/test-results/mixed-bounds-safety-with-assert.cpp

@@ -6,20 +6,24 @@
 #include "cpp2util.h"
 
 
+
 //=== Cpp2 type definitions and function declarations ===========================
 
 
 #line 2 "mixed-bounds-safety-with-assert.cpp2"
 [[nodiscard]] auto main() -> int;
+    
 
 #line 7 "mixed-bounds-safety-with-assert.cpp2"
 auto print_subrange(auto const& rng, cpp2::in<int> start, cpp2::in<int> end) -> void;
+    
 #line 19 "mixed-bounds-safety-with-assert.cpp2"
 
 #include <vector>
 #include <span>
 #include <iostream>
 
+
 //=== Cpp2 function definitions =================================================
 
 

regression-tests/test-results/mixed-captures-in-expressions-and-postconditions.cpp

@@ -6,6 +6,7 @@
 #include "cpp2util.h"
 
 
+
 //=== Cpp2 type definitions and function declarations ===========================
 
 
@@ -14,12 +15,14 @@
 
 #line 5 "mixed-captures-in-expressions-and-postconditions.cpp2"
 [[nodiscard]] auto main() -> int;
+    
 
 #line 17 "mixed-captures-in-expressions-and-postconditions.cpp2"
 extern std::vector<int> vec;
 
 auto insert_at(cpp2::in<int> where, cpp2::in<int> val) -> void;
 
+
 //=== Cpp2 function definitions =================================================
 
 

regression-tests/test-results/mixed-fixed-type-aliases.cpp

@@ -6,6 +6,7 @@
 #include "cpp2util.h"
 
 
+
 //=== Cpp2 type definitions and function declarations ===========================
 
 #include <typeinfo>
@@ -17,9 +18,11 @@ namespace my {
 
 #line 8 "mixed-fixed-type-aliases.cpp2"
 auto test(auto const& x) -> void;
+    
 
 #line 15 "mixed-fixed-type-aliases.cpp2"
 [[nodiscard]] auto main(int const argc_, char const* const* const argv_) -> int;
+    
 
 //=== Cpp2 function definitions =================================================
 

regression-tests/test-results/mixed-float-literals.cpp

@@ -6,6 +6,7 @@
 #include "cpp2util.h"
 
 
+
 //=== Cpp2 type definitions and function declarations ===========================
 
 void literals_cpp1() {
@@ -66,10 +67,12 @@ void literals_cpp1() {
 
 #line 57 "mixed-float-literals.cpp2"
 auto literals_cpp2() -> void;
+  
 
 #line 114 "mixed-float-literals.cpp2"
 [[nodiscard]] auto main() -> int;
 
+
 //=== Cpp2 function definitions =================================================
 
 

regression-tests/test-results/mixed-forwarding.cpp

@@ -6,6 +6,7 @@
 #include "cpp2util.h"
 
 
+
 //=== Cpp2 type definitions and function declarations ===========================
 
 #include <iostream>
@@ -25,12 +26,15 @@ auto use(auto const& x) -> void;
 
 // invoking each of these with an rvalue std::pair argument ...
 auto apply_implicit_forward(auto&& t) -> void;
+    
 
 #line 20 "mixed-forwarding.cpp2"
 auto apply_explicit_forward(auto&& t) -> void;
+    
 
 #line 25 "mixed-forwarding.cpp2"
 [[nodiscard]] auto main() -> int;
+    
 
 //=== Cpp2 function definitions =================================================
 

regression-tests/test-results/mixed-function-expression-and-std-for-each.cpp

@@ -6,6 +6,7 @@
 #include "cpp2util.h"
 
 
+
 //=== Cpp2 type definitions and function declarations ===========================
 
 #include <vector>
@@ -16,6 +17,7 @@
 
 #line 7 "mixed-function-expression-and-std-for-each.cpp2"
 [[nodiscard]] auto main() -> int;
+    
 
 //=== Cpp2 function definitions =================================================