Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix-#1201/refactor-design-smells #1202

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+48 −38
Open

fix-#1201/refactor-design-smells #1202

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 48 additions & 38 deletions core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,23 +32,47 @@

/** A wrapper around javac's lexer. */
final class JavacTokens {

/** The lexer eats terminal comments, so feed it one we don't care about. */
// TODO(b/33103797): fix javac and remove the work-around
private static final CharSequence EOF_COMMENT = "\n//EOF";

/** Represents a position in the source code with validation capabilities */
static final class TokenPosition {
private final int pos;
private final int endPos;

TokenPosition(int pos, int endPos) {
this.pos = pos;
this.endPos = endPos;
}

int getPos() {
return pos;
}

int getEndPos() {
return endPos;
}

void validateIndex(int index) {
checkArgument(
0 <= index && index < (endPos - pos),
"Expected %s in the range [0, %s)",
index,
endPos - pos);
}
}

/** An unprocessed input token, including whitespace and comments. */
static class RawTok {
private final String stringVal;
private final TokenKind kind;
private final int pos;
private final int endPos;
private final TokenPosition position;

RawTok(String stringVal, TokenKind kind, int pos, int endPos) {
this.stringVal = stringVal;
this.kind = kind;
this.pos = pos;
this.endPos = endPos;
this.position = new TokenPosition(pos, endPos);
}

/** The token kind, or {@code null} for whitespace and comments. */
Expand All @@ -58,12 +82,12 @@ public TokenKind kind() {

/** The start position. */
public int pos() {
return pos;
return position.getPos();
}

/** The end position. */
public int endPos() {
return endPos;
return position.getEndPos();
}

/** The escaped string value of a literal, or {@code null} for other tokens. */
Expand All @@ -74,7 +98,7 @@ public String stringVal() {

/** Lex the input and return a list of {@link RawTok}s. */
public static ImmutableList<RawTok> getTokens(
String source, Context context, Set<TokenKind> stopTokens) {
String source, Context context, Set<TokenKind> stopTokens) {
if (source == null) {
return ImmutableList.of();
}
Expand All @@ -94,7 +118,7 @@ public static ImmutableList<RawTok> getTokens(
tokens.add(new RawTok(null, null, last, c.getSourcePos(0)));
}
tokens.add(
new RawTok(null, null, c.getSourcePos(0), c.getSourcePos(0) + c.getText().length()));
new RawTok(null, null, c.getSourcePos(0), c.getSourcePos(0) + c.getText().length()));
last = c.getSourcePos(0) + c.getText().length();
}
}
Expand All @@ -108,11 +132,11 @@ public static ImmutableList<RawTok> getTokens(
tokens.add(new RawTok(null, null, last, t.pos));
}
tokens.add(
new RawTok(
t.kind == TokenKind.STRINGLITERAL ? "\"" + t.stringVal() + "\"" : null,
t.kind,
t.pos,
t.endPos));
new RawTok(
t.kind == TokenKind.STRINGLITERAL ? "\"" + t.stringVal() + "\"" : null,
t.kind,
t.pos,
t.endPos));
last = t.endPos;
} while (scanner.token().kind != TokenKind.EOF);
if (last < end) {
Expand All @@ -122,7 +146,7 @@ public static ImmutableList<RawTok> getTokens(
}

private static ImmutableList<CommentWithTextAndPosition> getComments(
Token token, Map<Comment, CommentWithTextAndPosition> comments) {
Token token, Map<Comment, CommentWithTextAndPosition> comments) {
if (token.comments == null) {
return ImmutableList.of();
}
Expand All @@ -132,7 +156,6 @@ private static ImmutableList<CommentWithTextAndPosition> getComments(

/** A {@link JavaTokenizer} that saves comments. */
static class CommentSavingTokenizer extends JavaTokenizer {

private final Map<Comment, CommentWithTextAndPosition> comments = new HashMap<>();

CommentSavingTokenizer(ScannerFactory fac, char[] buffer, int length) {
Expand All @@ -148,7 +171,7 @@ protected Comment processComment(int pos, int endPos, CommentStyle style) {
char[] buf = getRawCharactersReflectively(pos, endPos);
Comment comment = super.processComment(pos, endPos, style);
CommentWithTextAndPosition commentWithTextAndPosition =
new CommentWithTextAndPosition(pos, endPos, new String(buf));
new CommentWithTextAndPosition(pos, endPos, new String(buf));
comments.put(comment, commentWithTextAndPosition);
return comment;
}
Expand All @@ -162,10 +185,10 @@ private char[] getRawCharactersReflectively(int beginIndex, int endIndex) {
}
try {
return (char[])
instance
.getClass()
.getMethod("getRawCharacters", int.class, int.class)
.invoke(instance, beginIndex, endIndex);
instance
.getClass()
.getMethod("getRawCharacters", int.class, int.class)
.invoke(instance, beginIndex, endIndex);
} catch (ReflectiveOperationException e) {
throw new LinkageError(e.getMessage(), e);
}
Expand All @@ -174,30 +197,17 @@ private char[] getRawCharactersReflectively(int beginIndex, int endIndex) {

/** A {@link Comment} that saves its text and start position. */
static class CommentWithTextAndPosition {

private final int pos;
private final int endPos;
private final TokenPosition position;
private final String text;

public CommentWithTextAndPosition(int pos, int endPos, String text) {
this.pos = pos;
this.endPos = endPos;
this.position = new TokenPosition(pos, endPos);
this.text = text;
}

/**
* Returns the source position of the character at index {@code index} in the comment text.
*
* <p>The handling of javadoc comments in javac has more logic to skip over leading whitespace
* and '*' characters when indexing into doc comments, but we don't need any of that.
*/
public int getSourcePos(int index) {
checkArgument(
0 <= index && index < (endPos - pos),
"Expected %s in the range [0, %s)",
index,
endPos - pos);
return pos + index;
position.validateIndex(index);
return position.getPos() + index;
}

public String getText() {
Expand Down