Can now detect the line type

fern-pl · Apr 19, 2024 · 6d3c111 · 6d3c111
1 parent 05148d3
commit 6d3c111
Show file tree

Hide file tree

Showing 4 changed files with 92 additions and 54 deletions.
diff --git a/source/parsing/treegen/expressionParser.d b/source/parsing/treegen/expressionParser.d
diff --git a/source/parsing/treegen/scopeParser.d b/source/parsing/treegen/scopeParser.d
@@ -1,2 +1,61 @@
 module parsing.treegen.scopeParser;
+import parsing.tokenizer.tokens;
+import parsing.treegen.tokenRelationships;
 
+enum LineVariety
+{
+    SimpleExpression,
+    IfStatementWithScope,
+    IfStatementWithoutScope,
+    DeclarationLine,
+    DeclarationAndAssignment
+}
+
+struct LineVarietyAndLength
+{
+    LineVariety lineVariety;
+    size_t length;
+}
+
+LineVarietyAndLength getLineVarietyAndLength(Token[] tokens)
+{
+    size_t length;
+
+    static foreach (i, func; [
+            IfStatementWithScope,
+            IfStatementWithoutScope,
+            DeclarationLine,
+            DeclarationAndAssignment
+        ])
+    {
+        if (func.matchesToken(tokens, length))
+            return LineVarietyAndLength(
+                [
+                LineVariety.IfStatementWithScope,
+                LineVariety.IfStatementWithoutScope,
+                LineVariety.DeclarationLine,
+                LineVariety.DeclarationAndAssignment
+            ][i], length
+            );
+        length = 0;
+    }
+
+    return LineVarietyAndLength(LineVariety.SimpleExpression, -1);
+}
+
+void parseLine(Token[] tokens)
+{
+
+}
+
+unittest
+{
+    import std.stdio;
+    import parsing.tokenizer.make_tokens;
+
+    // assert(LineVariety.IfStatementWithoutScope == getLineVariety("if (hello) world;".tokenizeText));
+    // assert(LineVariety.IfStatementWithScope == getLineVariety("if (hello) {wo\n rl\nd};".tokenizeText));
+    getLineVarietyAndLength("int x = 4;".tokenizeText).writeln;
+    // DeclarationLine.matchesToken()
+
+}
diff --git a/source/parsing/treegen/tokenRelationships.d b/source/parsing/treegen/tokenRelationships.d
@@ -115,7 +115,7 @@ bool matchesToken(in TokenGrepPacket[] testWith, Token[] tokens)
 
 import std.stdio;
 
-private bool matchesToken(in TokenGrepPacket[] testWith, Token[] tokens, ref size_t index)
+bool matchesToken(in TokenGrepPacket[] testWith, Token[] tokens, ref size_t index)
 {
     foreach (testIndex, packet; testWith)
     {
@@ -200,8 +200,12 @@ private bool matchesToken(in TokenGrepPacket[] testWith, Token[] tokens, ref siz
                     braceDeph -= 1;
                 else if (braceDeph == 0)
                 {
-                    if (testWith[testIndex + 1 .. $].matchesToken(tokens[index .. $]))
+                    size_t index_inc;
+                    if (testWith[testIndex + 1 .. $].matchesToken(tokens[index .. $], index_inc))
+                    {
+                        index += index_inc;
                         return true;
+                    }
                 }
             }
             break;
@@ -219,22 +223,28 @@ unittest
     import parsing.tokenizer.make_tokens;
 
     assert(DeclarationLine.matchesToken(
-            tokenizeText("mod.type.submod x,r,q,a, A_variable  \n\r\t ;")));
+            tokenizeText("mod.type.submod x,r,q,a, A_variable  \n\r\t ;")
+    ));
     assert(DeclarationLine.matchesToken(tokenizeText("mod.type.submod x, a, e ,y;")));
     assert(!DeclarationLine.matchesToken(tokenizeText(";mod.type x;")));
     assert(!DeclarationLine.matchesToken(tokenizeText("123 mod.type x;")));
     assert(!DeclarationLine.matchesToken(tokenizeText("mod.type x = 5;")));
     assert(DeclarationAndAssignment.matchesToken(
-            tokenizeText("mod.type x, y, z  , o = someFunc();")));
+            tokenizeText("mod.type x, y, z  , o = someFunc();")
+    ));
     assert(!DeclarationAndAssignment.matchesToken(tokenizeText("someFunc();")));
     assert(!DeclarationLine.matchesToken(tokenizeText("someFunc();")));
     assert(IfStatementWithoutScope.matchesToken(tokenizeText("if (hello) testText;")));
     assert(IfStatementWithoutScope.matchesToken(tokenizeText("if (hello) v = ()=>print(1235);")));
     assert(IfStatementWithScope.matchesToken(tokenizeText("if (hello){}")));
     assert(IfStatementWithScope.matchesToken(tokenizeText("if (hello world){}")));
-    assert(IfStatementWithScope.matchesToken(tokenizeText(
-            "if (hello world){\n\n\r if(Some possible nested code) still works;}"
-        )));
+    assert(IfStatementWithScope.matchesToken(
+            tokenizeText(
+            "if (hello world){\n\n\r if(Some possible nested code) still works;}")
+    ));
+    assert(
+        DeclarationAndAssignment.matchesToken(tokenizeText("int x = 4;"))
+    );
 }
 
 enum OperatorOrder

diff --git a/source/parsing/treegen/treeGenUtils.d b/source/parsing/treegen/treeGenUtils.d
@@ -8,12 +8,14 @@ NameUnit genNameUnit(Token[] tokens, ref size_t index)
 {
     NameUnit ret;
     Nullable!Token tokenNullable = tokens.nextNonWhiteToken(index);
+    index--;
     Token token;
 
     // An attempt to generate a name at an EOF
     if (tokenNullable.ptr == null)
         return ret;
     token = tokenNullable;
+
     while (token.tokenVariety == TokenType.Letter || token.tokenVariety == TokenType.Period)
     {
 
@@ -27,6 +29,7 @@ NameUnit genNameUnit(Token[] tokens, ref size_t index)
             return ret;
 
         token = tokenNullable;
+
     }
     return ret;
 
@@ -82,3 +85,16 @@ AstNode[] parenthesisExtract(Token[] tokens)
     }
     return ret;
 }
+
+unittest
+{
+    import parsing.tokenizer.make_tokens;
+
+    size_t s = 0;
+    assert("int x = 4;".tokenizeText.genNameUnit(s).names == ["int".makeUnicodeString]);
+    s = 0;
+    assert("std.int x = 4;".tokenizeText.genNameUnit(s).names == [
+        "std".makeUnicodeString,
+        "int".makeUnicodeString
+        ]);
+}