Fixed grep bug where glob ignores closing braces

fern-pl · Apr 26, 2024 · 5e89089 · 5e89089
1 parent 108e041
commit 5e89089
Show file tree

Hide file tree

Showing 6 changed files with 26 additions and 34 deletions.
diff --git a/source/main.d b/source/main.d
@@ -11,20 +11,9 @@ void main()
 {
     size_t index = 0;
 
-    auto newScope = parseMultilineScope(GLOBAL_SCOPE_PARSE, "
-         int x = 693;
-         int main(){
-          int x = 4;
-         }
-        ".tokenizeText, index, nullable!ScopeData(null));
-    newScope.writeln;
-//       foreach (instruction; newScope.instructions)
-//       {
-//           instruction.tree;
-//       }
-//       "\n\nDeclared variables: ".writeln;
-//       foreach (var; newScope.declaredVariables)
-//       {
-//           var.writeln;
-//       }
+        import parsing.tokenizer.make_tokens;
+
+    auto t = "if (222) x.writeln;".tokenizeText;
+    auto r = IfStatementWithoutScope.matchesToken(t);
+    r.ptr.writeln;
 }
diff --git a/source/parsing/tokenizer/make_tokens.d b/source/parsing/tokenizer/make_tokens.d
@@ -66,6 +66,7 @@ private Token[] protoTokenize(string input)
             dchar[] comment = handleMultilineCommentsAtIndex(chars, index);
             if (comment.length != 0)
             {
+                index--;
                 tokens ~= Token(TokenType.Comment, comment, startingIndex);
                 continue;
             }

diff --git a/source/parsing/treegen/scopeParser.d b/source/parsing/treegen/scopeParser.d
@@ -301,10 +301,6 @@ unittest
                     ["float".makeUnicodeString]))
             ]
     );
-    foreach (i; newScope.instructions)
-    {
-            i.tree(-1);
-    }
 
     assert(newScope.instructions[0].action == AstAction.AssignVariable);
     assert(newScope.instructions[1].action == AstAction.AssignVariable);
@@ -339,3 +335,12 @@ unittest
             "floaty".makeUnicodeString
         ]);
 }
+unittest
+{
+    import parsing.tokenizer.make_tokens;
+
+    size_t index;
+    auto t = "let x = 4/*asdadasd*/;".tokenizeText;
+    auto r = getLineVarietyTestResult(FUNCTION_SCOPE_PARSE, t, index);
+    assert(r.lineVariety == LineVariety.DeclarationAndAssignment);
+}
diff --git a/source/parsing/treegen/tokenRelationships.d b/source/parsing/treegen/tokenRelationships.d
@@ -383,17 +383,14 @@ Nullable!(TokenGrepResult[]) matchesToken(in TokenGrepPacket[] testWith, Token[]
 
                 if (token.tokenVariety == TokenType.OpenBraces)
                     braceDeph += 1;
-                else if (token.tokenVariety == TokenType.CloseBraces){
+                else if (token.tokenVariety == TokenType.CloseBraces && braceDeph != 0)
                     braceDeph -= 1;
-                    if (braceDeph == -1)
-                        return tokenGrepBox(null);
-                }else if (braceDeph == 0)
+                else if (braceDeph == 0)
                 {
                     size_t index_inc = 0;
                     auto res = grepMatchGroup.matchesToken(tokens[index .. $], index_inc);
                     if (res.ptr)
                     {
-
                         globResult.tokens = tokens[startingIndex .. index];
                         index += index_inc;
                         return tokenGrepBox(returnVal ~ globResult ~ res.value);

diff --git a/source/tests/tokenizer.d b/source/tests/tokenizer.d
@@ -22,6 +22,7 @@ unittest
         dchar[] output = handleMultilineCommentsAtIndex(stringWithComment, index);
 
         assert(output == commentFromString);
+        import std.stdio;
         assert(stringWithComment[index .. $] == afterComment);
 
     }

diff --git a/source/tests/treegen.d b/source/tests/treegen.d
@@ -60,15 +60,14 @@ unittest
     ));
     assert(null == DeclarationAndAssignment.matchesToken(tokenizeText("someFunc();")));
     assert(null == DeclarationLine.matchesToken(tokenizeText("someFunc();")));
-    // TODO: FUCK
-    // assert(null != IfStatementWithoutScope.matchesToken(tokenizeText("if (hello) testText;")));
-    // assert(null !=IfStatementWithoutScope.matchesToken(tokenizeText("if (hello) v = ()=>print(1235);")));
-    // assert(null !=IfStatementWithScope.matchesToken(tokenizeText("if (hello){}")));
-    // assert(null !=IfStatementWithScope.matchesToken(tokenizeText("if (hello world){}")));
-    // assert(null !=IfStatementWithScope.matchesToken(
-    //         tokenizeText(
-    //         "if (hello world){\n\n\r if(Some possible nested code) still works;}")
-    // ));
+    assert(null != IfStatementWithoutScope.matchesToken(tokenizeText("if (hello) testText;")));
+    assert(null !=IfStatementWithoutScope.matchesToken(tokenizeText("if (hello) v = ()=>print(1235);")));
+    assert(null !=IfStatementWithScope.matchesToken(tokenizeText("if (hello){}")));
+    assert(null !=IfStatementWithScope.matchesToken(tokenizeText("if (hello world){}")));
+    assert(null !=IfStatementWithScope.matchesToken(
+            tokenizeText(
+            "if (hello world){\n\n\r if(Some possible nested code) still works;}")
+    ));
     assert( null !=
         DeclarationAndAssignment.matchesToken(tokenizeText("int x = 4;"))
     );