From d4149d21e9dbc61ca60173f94bc7b9b6a2796b65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Le=C3=B3n=20Orell=20Valerian=20Liehr?= <me@fmease.dev>
Date: Fri, 4 Aug 2023 23:44:11 +0200
Subject: [PATCH] Allow closing brackets to be further dedented

Previously, we would insert a line break before such a closing brackets
according to the rules. However, I'd like to be able to place my closing
bracket at the same indentation level as the respective opening one.

See the updated UI tests for example.

The current approach is quite liberal and careless. In the future we
should probably restrict this a bit, e.g. forbidding closing brackets
to be further indented than the respective opening one.
---
 compiler/lexer/src/lib.rs                     | 26 ++++++++++++++++---
 compiler/lexer/src/test.rs                    | 23 +++++++++++++---
 library/core/source/library/ordering.lushui   |  8 +++---
 library/core/source/library/vector.lushui     |  6 +++--
 test/ui/analyze-types.lushui                  |  2 +-
 test/ui/any.lushui                            |  2 +-
 test/ui/factorial.lushui                      |  2 +-
 test/ui/intrinsic.lushui                      |  2 +-
 test/ui/lengthy-type-mismatch.lushui          |  2 +-
 test/ui/lowering/use-path-trees.lushui        |  2 +-
 .../exposure/undefined-reach.lushui           |  2 +-
 .../sequence-literals-unsupported.lushui      |  2 +-
 test/ui/numbers.lushui                        |  2 +-
 .../transitive-dependencies/main.lushui       |  9 +++++--
 test/ui/parsing/case-analysis.lushui          |  2 +-
 test/ui/punctuation.lushui                    |  6 ++++-
 test/ui/running-intrinsic-functions.lushui    |  6 +++--
 test/ui/use-bindings-are-aliases.lushui       |  2 +-
 18 files changed, 78 insertions(+), 28 deletions(-)

diff --git a/compiler/lexer/src/lib.rs b/compiler/lexer/src/lib.rs
index 2af244e6..c9d26826 100644
--- a/compiler/lexer/src/lib.rs
+++ b/compiler/lexer/src/lib.rs
@@ -301,10 +301,13 @@ impl<'a> Lexer<'a> {
             .map_or(false, |token| token.bare.introduces_indented_section());
 
         // Squash consecutive line breaks into a single one.
+        // This leads to more legible and fewer diagnostics later in the parser in case the
+        // line break wasn't expected. Further, it might lead to less churn in the parser.
         self.take_while(|character| character == '\n');
-        // Tentatively register the line break. If certain conditions are met
-        // later on (*), we will remove it again.
+        // Tentatively register the line break.
+        // If certain conditions are met later on (*), we will remove it again.
         self.add(LineBreak);
+        let mut has_removed_line_break = false;
 
         self.local_span = self
             .index()
@@ -313,7 +316,10 @@ impl<'a> Lexer<'a> {
         let mut spaces = Spaces(0);
         self.take_while_with(|character| character == ' ', || spaces.0 += 1);
 
-        // If the line is empty ignore it. This is important for indented comments.
+        // If the line is empty ignore it. While most places in the parser can deal with
+        // “empty declarations”, in the future (once `line_is_empty` can detect comments
+        // as well) we might want to simplify the parser by throwing away the special
+        // handling (i.e. skipping) of empty lines.
         if self.line_is_empty() {
             spaces = self.indentation;
         }
@@ -336,6 +342,7 @@ impl<'a> Lexer<'a> {
             if change == Ordering::Greater {
                 // (*) Remove the line break again.
                 self.tokens.pop();
+                has_removed_line_break = true;
                 self.add(BareToken::Indentation);
                 self.sections.enter(Section::Indented {
                     brackets: self.brackets.stack.len(),
@@ -355,6 +362,7 @@ impl<'a> Lexer<'a> {
             {
                 // (*) Remove the line break again.
                 self.tokens.pop();
+                has_removed_line_break = true;
             }
 
             if change == Ordering::Greater {
@@ -365,11 +373,12 @@ impl<'a> Lexer<'a> {
         if change == Ordering::Less {
             // Remove syntactically legal but superfluous line breaks that
             // come before dedendentation (which also act as terminators).
-            // @Question is this still reachable???
             if self.sections.current_continued().0.is_indented()
                 && let Some(Spanned!(_, BareToken::LineBreak)) = self.tokens.last()
             {
+                // (*) Remove the line break again.
                 self.tokens.pop();
+                has_removed_line_break = true;
             }
 
             for _ in 0..indentation.0 {
@@ -384,6 +393,15 @@ impl<'a> Lexer<'a> {
         }
 
         self.indentation = spaces;
+
+        // @Task only remove the line break if the indentation of the closing bracket is
+        // greater or equal to the one of the corresponding opening bracket.
+        // @Note actually it would be even better if we could emit a nice diagnostic for
+        // closing brackets that are dedented too far.
+        if let Some(')' | ']' | '}') = self.peek() && !has_removed_line_break {
+            // (*) Remove the line break again.
+            self.tokens.pop();
+        }
     }
 
     fn line_is_empty(&mut self) -> bool {
diff --git a/compiler/lexer/src/test.rs b/compiler/lexer/src/test.rs
index 514c82ff..e8157632 100644
--- a/compiler/lexer/src/test.rs
+++ b/compiler/lexer/src/test.rs
@@ -607,7 +607,7 @@ of
 }
 
 #[test]
-fn no_superfluous_virtual_semicolon_before_virtual_curly_bracket_with_continued_section() {
+fn no_superfluous_line_break_before_dedentation_token_with_continued_section() {
     assert_lex_eq!(
         "\
 of
@@ -627,7 +627,7 @@ of
 }
 
 #[test]
-fn empty_indented_section_does_not_create_virtual_curly_brackets() {
+fn empty_indented_section_does_not_create_indentation_tokens() {
     assert_lex_eq!(
         "\
 of
@@ -694,7 +694,6 @@ fn round_bracket_closes_indented_section() {
             Token::new(span(14, 16), Of),
             Token::new(span(17, 21), Indentation),
             Token::new(span(21, 23), Word("fo".into())),
-            Token::new(span(23, 24), LineBreak),
             // @Question better span?
             Token::new(span(28, 29), Dedentation),
             Token::new(span(28, 29), ClosingRoundBracket),
@@ -777,6 +776,24 @@ fn brackets_reset_indentation() {
     )
 }
 
+#[test]
+fn dedented_closing_bracket_does_not_create_line_break() {
+    assert_lex_eq!(
+        "\
+hook = (
+    element
+)",
+        vec![
+            Token::new(span(1, 5), Word("hook".into())),
+            Token::new(span(6, 7), Equals),
+            Token::new(span(8, 9), OpeningRoundBracket),
+            Token::new(span(14, 21), Word("element".into())),
+            Token::new(span(22, 23), ClosingRoundBracket),
+            Token::new(span(23, 23), EndOfInput),
+        ]
+    );
+}
+
 #[test]
 fn indentation_at_start_of_input() {
     assert_lex_eq!(
diff --git a/library/core/source/library/ordering.lushui b/library/core/source/library/ordering.lushui
index 3d16f7bf..12a5d1e9 100644
--- a/library/core/source/library/ordering.lushui
+++ b/library/core/source/library/ordering.lushui
@@ -1,6 +1,11 @@
 ;; Types and functions related to comparing and ordering.
 module
 
+use topmost.(
+    unit.(unit, Unit, Thunk, force),
+    int.Int,
+)
+
 @public
 data Ordering of
     less
@@ -17,9 +22,6 @@ invert (o: Ordering): Ordering =
         equal => equal
         greater => less
 
-use topmost.unit.(unit, Unit, Thunk, force)
-use topmost.int.Int
-
 ;;; @Task uncomment once `@lazy` is implemented
 ;;; @public
 ;;; then (o: Ordering) @lazy (p: Thunk Ordering): Ordering =
diff --git a/library/core/source/library/vector.lushui b/library/core/source/library/vector.lushui
index 73396505..48cbbd35 100644
--- a/library/core/source/library/vector.lushui
+++ b/library/core/source/library/vector.lushui
@@ -1,8 +1,10 @@
 ;; Homogeneous lists with a fixed length and related functions.
 module
 
-use topmost.type.Type
-use topmost.nat.(Nat, +)
+use topmost.(
+    type.Type,
+    nat.(Nat, +),
+)
 
 @public @known
 data Vector A: For (n: Nat) -> Type of
diff --git a/test/ui/analyze-types.lushui b/test/ui/analyze-types.lushui
index 279a40ae..4ef3efdf 100644
--- a/test/ui/analyze-types.lushui
+++ b/test/ui/analyze-types.lushui
@@ -1,7 +1,7 @@
 use extern.core.(
     type.Type,
     bool.(false, true, Bool),
-    )
+)
 
 is-nat32-or-nat64 (A: Type -> Type): Bool =
     case A of
diff --git a/test/ui/any.lushui b/test/ui/any.lushui
index c07e7d6a..c0257bab 100644
--- a/test/ui/any.lushui
+++ b/test/ui/any.lushui
@@ -5,7 +5,7 @@ use extern.core.(
     list.List,
     bool.(Bool, false, true),
     unit.(Unit, unit),
-    )
+)
 
 data Any: Type of
     any: For (A: Type) -> A -> Any
diff --git a/test/ui/factorial.lushui b/test/ui/factorial.lushui
index b9440a5c..5fff5d0e 100644
--- a/test/ui/factorial.lushui
+++ b/test/ui/factorial.lushui
@@ -3,7 +3,7 @@
 use extern.core.(
     bool.(self, Bool, false, true),
     nat.(self, Nat),
-    )
+)
 
 main: Nat = factorial 20
 ;;; main: Nat = factorial 50 ;;; @Note takes several minutes after a rustc update (compared to a second)
diff --git a/test/ui/intrinsic.lushui b/test/ui/intrinsic.lushui
index 2a51b7ea..8a26695b 100644
--- a/test/ui/intrinsic.lushui
+++ b/test/ui/intrinsic.lushui
@@ -6,7 +6,7 @@ use extern.core.(
     bool.Bool,
     list.List,
     tuple.Tuple,
-    )
+)
 
 combine: Nat -> Nat -> Nat = nat.+
 successor: Nat -> Nat = combine 1
diff --git a/test/ui/lengthy-type-mismatch.lushui b/test/ui/lengthy-type-mismatch.lushui
index 29069f29..691080a2 100644
--- a/test/ui/lengthy-type-mismatch.lushui
+++ b/test/ui/lengthy-type-mismatch.lushui
@@ -5,7 +5,7 @@ use extern.core.(
     int.Int,
     unit.Unit,
     function.identity,
-    )
+)
 
 it: Tuple List.((Type), Int, Unit, Int) =
     Tuple.prepend
diff --git a/test/ui/lowering/use-path-trees.lushui b/test/ui/lowering/use-path-trees.lushui
index 3231aa85..642991db 100644
--- a/test/ui/lowering/use-path-trees.lushui
+++ b/test/ui/lowering/use-path-trees.lushui
@@ -19,4 +19,4 @@ use foo.(A as F1, B as F2)
 use topmost.foo.alpha.(
     gamma.delta.zeta.(foo, bar),
     xoo.moo.doo as XXX
-    )
+)
diff --git a/test/ui/name-resolution/exposure/undefined-reach.lushui b/test/ui/name-resolution/exposure/undefined-reach.lushui
index f3eeff9c..b9115e7d 100644
--- a/test/ui/name-resolution/exposure/undefined-reach.lushui
+++ b/test/ui/name-resolution/exposure/undefined-reach.lushui
@@ -18,7 +18,7 @@ use topmost.(
     namespaced as indir0,
     indir0 as indir1,
     indir1 as indir2,
-    )
+)
 
 @(public self.indir2.emptiness)
 X: Type = Unit
diff --git a/test/ui/name-resolution/sequence-literals-unsupported.lushui b/test/ui/name-resolution/sequence-literals-unsupported.lushui
index ca677409..3317c567 100644
--- a/test/ui/name-resolution/sequence-literals-unsupported.lushui
+++ b/test/ui/name-resolution/sequence-literals-unsupported.lushui
@@ -2,7 +2,7 @@ use extern.core.(
     list.List,
     nat32.Nat32,
     text.Text,
-    )
+)
 
 construct: List = List.(1, 3, 100, Nat32.0)
 
diff --git a/test/ui/numbers.lushui b/test/ui/numbers.lushui
index 85f7a8d0..b9e919fe 100644
--- a/test/ui/numbers.lushui
+++ b/test/ui/numbers.lushui
@@ -5,7 +5,7 @@ use extern.core.(
     nat32.Nat32,
     int.Int,
     int64.Int64,
-    )
+)
 
 alpha: Int = Int.-1000
 
diff --git a/test/ui/packages/transitive-dependencies/main.lushui b/test/ui/packages/transitive-dependencies/main.lushui
index 3cea597b..73747761 100644
--- a/test/ui/packages/transitive-dependencies/main.lushui
+++ b/test/ui/packages/transitive-dependencies/main.lushui
@@ -1,7 +1,12 @@
 ;;; TEST auxiliary packages/transitive-dependencies
 
-use extern.core.(type.Type, nat.(Nat, ==), bool.Bool)
-use extern.core.(tuple.Tuple, list.List)
+use extern.core.(
+    type.Type,
+    nat.(Nat, ==),
+    bool.Bool,
+    tuple.Tuple,
+    list.List,
+)
 use extern.alpha.(bet, g-one, jamma)
 use extern.gamma.(G as The-G, g1, gam)
 
diff --git a/test/ui/parsing/case-analysis.lushui b/test/ui/parsing/case-analysis.lushui
index cb2f6e82..0c12662a 100644
--- a/test/ui/parsing/case-analysis.lushui
+++ b/test/ui/parsing/case-analysis.lushui
@@ -24,7 +24,7 @@ main: Untyped =
 use extern.core.(
     nat.Nat,
     bool.Bool.(self, false, true),
-    )
+)
 
 invert (b: Bool): Bool =
     case b of
diff --git a/test/ui/punctuation.lushui b/test/ui/punctuation.lushui
index b60e3dd5..437ca2d4 100644
--- a/test/ui/punctuation.lushui
+++ b/test/ui/punctuation.lushui
@@ -1,6 +1,10 @@
 ;;; TEST pass run
 
-use extern.core.nat.(Nat, add as +, multiply as *)
+use extern.core.nat.(
+    Nat,
+    add as +,
+    multiply as *,
+)
 
 x: Nat = * 1 2
 
diff --git a/test/ui/running-intrinsic-functions.lushui b/test/ui/running-intrinsic-functions.lushui
index 0ac21439..5ad8d028 100644
--- a/test/ui/running-intrinsic-functions.lushui
+++ b/test/ui/running-intrinsic-functions.lushui
@@ -1,7 +1,9 @@
 ;;; TEST pass run
 
-use extern.core.text.(concat, Text)
-use extern.core.nat.(self, Nat)
+use extern.core.(
+    text.(concat, Text),
+    nat.(self, Nat),
+)
 
 base: Nat = 1034032
 
diff --git a/test/ui/use-bindings-are-aliases.lushui b/test/ui/use-bindings-are-aliases.lushui
index e0cd2cc0..80ff70f3 100644
--- a/test/ui/use-bindings-are-aliases.lushui
+++ b/test/ui/use-bindings-are-aliases.lushui
@@ -4,7 +4,7 @@ use extern.core.(
     type.Type,
     nat.(Nat, Nat as N),
     list.List,
-    )
+)
 
 data Source: Type of
     source: N -> Source