Skip to content

Commit

Permalink
strip out comments before tokenizing
Browse files Browse the repository at this point in the history
  • Loading branch information
PhilipNelson5 committed Apr 2, 2024
1 parent 9c6ed72 commit eedafd2
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 1 deletion.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
## Unreleased

- Added doxygen format support ([#120](https://github.com/NilsJPWerner/autoDocstring/issues/120)) (@daluar @PhilipNelson5)
- Handle comments in multiline definitions ([#252](https://github.com/NilsJPWerner/autoDocstring/issues/252)) (@GabrielSchoenweiler @PhilipNelson5)

[All Changes](https://github.com/NilsJPWerner/autoDocstring/compare/v0.6.1...master)

Expand Down
18 changes: 17 additions & 1 deletion src/parse/tokenize_definition.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ export function tokenizeDefinition(functionDefinition: string): string[] {
return [];
}

const tokens = tokenizeParameterString(match[1]);
const tokens = tokenizeParameterString(stripComments(match[1]));

if (match[2] != undefined) {
tokens.push(match[2]);
Expand All @@ -16,6 +16,22 @@ export function tokenizeDefinition(functionDefinition: string): string[] {
return tokens;
}

function stripComments(parameterString: string): string {
let cleanString = "";
let position = 0;

while (position < parameterString.length) {
// When a comment is encountered, skip ahead to the end of the line
if (parameterString[position] === "#") {
position = parameterString.indexOf("\n", position);
}

cleanString += parameterString[position++];
}

return cleanString;
}

function tokenizeParameterString(parameterString: string): string[] {
const stack: string[] = [];
const parameters: string[] = [];
Expand Down
20 changes: 20 additions & 0 deletions src/test/parse/tokenize_definition.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,4 +113,24 @@ describe("tokenizeDefinition()", () => {

expect(result).to.have.ordered.members(["arg", "arg_2"]);
});

it("should ignore comments in the middle of the function definition", () => {
const functionDefinition = `def abc_c(
arg, # comment
arg_2, # comment, comment
arg_3 # comment with special characters "'"({[]})
):`;
const result = tokenizeDefinition(functionDefinition);

expect(result).to.have.ordered.members(["arg", "arg_2", "arg_3"]);
});
it("should ignore comments in the middle of the class definition", () => {
const functionDefinition = `class abc_c(
arg, # comment,
arg_2 # comment with special characters "'"({[]})
):`;
const result = tokenizeDefinition(functionDefinition);

expect(result).to.have.ordered.members(["arg", "arg_2"]);
});
});

0 comments on commit eedafd2

Please sign in to comment.