Skip to content

Commit

Permalink
Dfmt code and clean some comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@HeronErin
HeronErin committed Apr 25, 2024
1 parent a1b1fbe commit ebac94b
Show file tree
Hide file tree
Showing 5 changed files with 75 additions and 89 deletions.
11 changes: 0 additions & 11 deletions source/main.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,4 @@ void main()
// auto scope_ = tokens.parseMultilineScope(index, nullable!ScopeData(null));
// import std.stdio;
// scope_.declaredVariables.writeln;
import parsing.tokenizer.make_tokens;
import parsing.treegen.scopeParser;
import std.stdio;

size_t index = 0;
auto scopeData = new ScopeData;
parseLine("int x, y, z = 4*5+2;".tokenizeText, index, scopeData);

"Vars: ".write;
scopeData.declaredVariables.writeln;
scopeData.instructions[0].tree(-1);
}
122 changes: 62 additions & 60 deletions source/parsing/treegen/scopeParser.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,22 +69,24 @@ LineVarietyTestResult getLineVarietyTestResult(Token[] tokens, size_t index)
DeclarationLine,
DeclarationAndAssignment
])
{{
Nullable!(TokenGrepResult[]) grepResults = func.matchesToken(tokens, temp_index);
if (null != grepResults)
return LineVarietyTestResult(
[
LineVariety.TotalImport,
LineVariety.SelectiveImport,
LineVariety.ModuleDeclaration,
LineVariety.IfStatementWithScope,
LineVariety.IfStatementWithoutScope,
LineVariety.DeclarationLine,
LineVariety.DeclarationAndAssignment
][i], temp_index - index, grepResults.value
);
temp_index = index;
}}
{
{
Nullable!(TokenGrepResult[]) grepResults = func.matchesToken(tokens, temp_index);
if (null != grepResults)
return LineVarietyTestResult(
[
LineVariety.TotalImport,
LineVariety.SelectiveImport,
LineVariety.ModuleDeclaration,
LineVariety.IfStatementWithScope,
LineVariety.IfStatementWithoutScope,
LineVariety.DeclarationLine,
LineVariety.DeclarationAndAssignment
][i], temp_index - index, grepResults.value
);
temp_index = index;
}
}

return LineVarietyTestResult(LineVariety.SimpleExpression, -1);
}
Expand Down Expand Up @@ -143,36 +145,38 @@ LineVarietyTestResult parseLine(Token[] tokens, ref size_t index, ScopeData pare
break;
case LineVariety.SelectiveImport:
size_t endingIndex = index + lineVariety.length;
scope (exit) index = endingIndex;
scope (exit)
index = endingIndex;

auto statement = ImportStatement(
keywords,
lineVariety.tokenMatches[IMPORT_PACKAGE_NAME].name,
[]
);

statement.importSelection ~= lineVariety.tokenMatches[SELECTIVE_IMPORT_SELECTIONS]
.commaSeperated
.collectNameUnits();
statement.importSelection ~= lineVariety
.tokenMatches[SELECTIVE_IMPORT_SELECTIONS]
.commaSeperated
.collectNameUnits();

parent.imports ~= statement;
break;
case LineVariety.DeclarationLine:
case LineVariety.DeclarationAndAssignment:
size_t endingIndex = index + lineVariety.length;
scope (exit) index = endingIndex;
scope (exit)
index = endingIndex;

NameUnit declarationType = lineVariety.tokenMatches[DECLARATION_TYPE].name;
NameUnit[] declarationNames = lineVariety.tokenMatches[DECLARATION_VARS].commaSeperated.collectNameUnits();
lineVariety.tokenMatches[1].commaSeperated.writeln;
foreach (NameUnit name; declarationNames)
parent.declaredVariables ~= DeclaredVariable(name, declarationType);

if (lineVariety.lineVariety == LineVariety.DeclarationLine) break;


if (lineVariety.lineVariety == LineVariety.DeclarationLine)
break;

auto nodes = lineVariety.tokenMatches[DECLARATION_EXPRESSION].tokens.expressionNodeFromTokens();

// nodes.data.writeln;

if (nodes.length != 1)
throw new SyntaxError(
"Expression node tree could not be parsed properly (Not reducable into single node)");
Expand All @@ -181,22 +185,21 @@ LineVarietyTestResult parseLine(Token[] tokens, ref size_t index, ScopeData pare
assignment.action = AstAction.AssignVariable;
assignment.assignVariableNodeData.name = declarationNames;
assignment.assignVariableNodeData.value = result;

parent.instructions ~= assignment;

break;
case LineVariety.SimpleExpression:
size_t expression_end = tokens.findNearestSemiColon(index);
if (expression_end == -1)
throw new SyntaxError("Semicolon not found!");
auto nodes = expressionNodeFromTokens(tokens[index .. expression_end]);
// tokens[index .. expression_end].writeln;
if (nodes.length != 1)
throw new SyntaxError(
"Expression node tree could not be parsed properly (Not reducable into single node)");
parent.instructions ~= nodes[0];
index = expression_end + 1;

break;
default:
import std.conv;
Expand All @@ -211,36 +214,35 @@ ScopeData parseMultilineScope(Token[] tokens, ref size_t index, Nullable!ScopeDa
{
ScopeData scopeData = new ScopeData;
scopeData.parent = parent;
parseLine(tokens, index, scopeData).lineVariety.writeln;
parseLine(tokens, index, scopeData).lineVariety.writeln;
parseLine(tokens, index, scopeData).lineVariety.writeln;
parseLine(tokens, index, scopeData).lineVariety.writeln;
parseLine(tokens, index, scopeData).lineVariety.writeln;
parseLine(tokens, index, scopeData).lineVariety.writeln;
// parseLine(tokens, index, scopeData).lineVariety.writeln;
// parseLine(tokens, index, scopeData).lineVariety.writeln;
while (index < tokens.length)
{
LineVarietyTestResult lineData = parseLine(tokens, index, scopeData);
Nullable!Token testToken = tokens.nextNonWhiteToken(index);
if (testToken == null)
break;
index--;

}

return scopeData;
}

// unittest
// {
// import parsing.tokenizer.make_tokens;
// import parsing.treegen.scopeParser;

// size_t index = 0;
// auto newScope = parseMultilineScope("
// int x, y;
// x = 5;
// y = 1;
// x = 3;
// int tv = x++ + y;
// float floaty = tv / 2;
// int xx;
// int xxx;
// ".tokenizeText(), index, nullable!ScopeData(null));
// newScope.declaredVariables.writeln;

// foreach (x ; newScope.instructions)
// x.tree(-1);
// }
unittest
{
import parsing.tokenizer.make_tokens;
import parsing.treegen.scopeParser;

size_t index = 0;
auto newScope = parseMultilineScope("
int x, y;
x = 5;
y = 1;
x = 3;
int tv = x++ + y;
float floaty = tv / 2;
".tokenizeText(), index, nullable!ScopeData(null));
newScope.declaredVariables.writeln;

foreach (x; newScope.instructions)
x.tree(-1);
}
21 changes: 11 additions & 10 deletions source/parsing/treegen/tokenRelationships.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ Nullable!(TokenGrepResult[]) matchesToken(in TokenGrepPacket[] testWith, Token[]
size_t index = 0;
return matchesToken(testWith, tokens, index);
}

alias tokenGrepBox = Nullable!(TokenGrepResult[]);
Nullable!(TokenGrepResult[]) matchesToken(in TokenGrepPacket[] testWith, Token[] tokens, ref size_t index)
{
Expand Down Expand Up @@ -266,11 +267,10 @@ Nullable!(TokenGrepResult[]) matchesToken(in TokenGrepPacket[] testWith, Token[]
TokenGrepResult globResult;
globResult.method = TokenGrepMethod.Glob;
globResult.tokens = [];

if (firstGlob.ptr)
return tokenGrepBox(returnVal ~ globResult ~ firstGlob.value);


return tokenGrepBox(returnVal ~ globResult ~ firstGlob.value);

int braceDeph = 0;
size_t startingIndex = index;
auto grepMatchGroup = testWith[testIndex + 1 .. $];
Expand All @@ -281,7 +281,7 @@ Nullable!(TokenGrepResult[]) matchesToken(in TokenGrepPacket[] testWith, Token[]
return tokenGrepBox(null);
Token token = tokenNullable;
globResult.tokens ~= token;

if (token.tokenVariety == TokenType.OpenBraces)
braceDeph += 1;
else if (token.tokenVariety == TokenType.CloseBraces && braceDeph != 0)
Expand All @@ -292,9 +292,9 @@ Nullable!(TokenGrepResult[]) matchesToken(in TokenGrepPacket[] testWith, Token[]
auto res = grepMatchGroup.matchesToken(tokens[index .. $], index_inc);
if (res.ptr)
{

globResult.tokens = tokens[startingIndex .. index];

index += index_inc;
return tokenGrepBox(returnVal ~ globResult ~ res.value);
}
Expand All @@ -310,16 +310,17 @@ Nullable!(TokenGrepResult[]) matchesToken(in TokenGrepPacket[] testWith, Token[]
return tokenGrepBox(returnVal);
}

NameUnit[] collectNameUnits(TokenGrepResult[] greps){
NameUnit[] collectNameUnits(TokenGrepResult[] greps)
{
NameUnit[] ret;
foreach (TokenGrepResult grepResult ; greps){
foreach (TokenGrepResult grepResult; greps)
{
assert(grepResult.method == TokenGrepMethod.NameUnit);
ret ~= grepResult.name;
}
return ret;
}


enum OperatorOrder
{
LeftToRight,
Expand Down
8 changes: 2 additions & 6 deletions source/parsing/treegen/treeGenUtils.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,8 @@ NameUnit genNameUnit(Token[] tokens, ref size_t index)
.Period)
{

if (token.tokenVariety != TokenType.Period){
// dchar[] dataCopy = new dchar[token.value.length];
// dataCopy[0..$] = token.value;
// import std.stdio;
// token.value.writeln;
// dataCopy.ptr.writeln;
if (token.tokenVariety != TokenType.Period)
{
nameData ~= token.value;
}

Expand Down
2 changes: 0 additions & 2 deletions source/tests/treegen.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,5 @@ unittest
size_t index = 0;
auto scopeData = new ScopeData;
parseLine("int x, y, z = 4*5+2;".tokenizeText, index, scopeData);
scopeData.declaredVariables[0].name.names.length.writeln;
scopeData.declaredVariables[0].name.names[0].writeln;
}

0 comments on commit ebac94b

Please sign in to comment.