Formatting on Enter (#649)

* Basic tokenizer

* Fixed property names

* Tests, round I

* Tests, round II

* tokenizer test

* Remove temorary change

* Fix merge issue

* Merge conflict

* Merge conflict

* Completion test

* Fix last line

* Fix javascript math

* Make test await for results

* Add license headers

* Rename definitions to types

* License headers

* Fix typo in completion details (typo)

* Fix hover test

* Russian translations

* Update to better translation

* Fix typo

*  #70 How to get all parameter info when filling in a function param list

* Fix #70 How to get all parameter info when filling in a function param list

* Clean up

* Clean imports

* CR feedback

* Trim whitespace for test stability

* More tests

* Better handle no-parameters documentation

* Better handle ellipsis and Python3

* #385 Auto-Indentation doesn't work after comment

* #141 Auto indentation broken when return keyword involved

* Undo changes

* On type formatting

* Fix warnings

* Round I

* Round 2

* Round 3

* Round 4

* Round 5

* no message

* Round 6

* Round 7

* Clean up targets and messages

* Settings propagation

* Tests

* Test warning

* Fix installer tests

* Tests

* Test fixes

* Fix terminal service and tests async/await

* Fix mock setup

* Test fix

* Test async/await fix

* Test fix + activate tslint on awaits

* Use command manager

* Work around updateSettings

* Multiroot fixes, partial

* More workarounds

* Multiroot tests

* Fix installer test

* Test fixes

* Disable prospector

* Enable dispose in all cases

* Fix event firing

* Min pylint options

* Min checkers & pylintrc discovery

* Fix Windows path in tests for Travis

* Fix Mac test

* Test fix

* Work around VSC issue with formatting on save #624

* Workaround test

* Unused

* Old file

* Brace and colon handling

* More tests

* Don't format inside strings and comments

* Provider tests

* Remove duplicate code

package.json

@@ -1594,6 +1594,7 @@
  "tree-kill": "^1.1.0",
  "typescript-char": "^0.0.0",
  "uint64be": "^1.0.1",
+ "unicode": "^10.0.0",
  "untildify": "^3.0.2",
  "vscode-debugadapter": "^1.0.1",
  "vscode-debugprotocol": "^1.0.1",

src/client/extension.ts

@@ -57,6 +57,7 @@ import { StopWatch } from './telemetry/stopWatch';
 import { registerTypes as commonRegisterTerminalTypes } from './terminals/serviceRegistry';
 import { ICodeExecutionManager } from './terminals/types';
 import { BlockFormatProviders } from './typeFormatters/blockFormatProvider';
+import { OnEnterFormatter } from './typeFormatters/onEnterFormatter';
 import { TEST_OUTPUT_CHANNEL } from './unittests/common/constants';
 import * as tests from './unittests/main';
 import { registerTypes as unitTestsRegisterTypes } from './unittests/serviceRegistry';
@@ -190,6 +191,8 @@ export async function activate(context: vscode.ExtensionContext) {
  context.subscriptions.push(new WorkspaceSymbols(serviceContainer));
 
  context.subscriptions.push(vscode.languages.registerOnTypeFormattingEditProvider(PYTHON, new BlockFormatProviders(), ':'));
+ context.subscriptions.push(vscode.languages.registerOnTypeFormattingEditProvider(PYTHON, new OnEnterFormatter(), '\n'));
+
  // In case we have CR LF
  const triggerCharacters: string[] = os.EOL.split('');
  triggerCharacters.shift();

src/client/formatters/lineFormatter.ts

@@ -0,0 +1,118 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// tslint:disable-next-line:import-name
+import Char from 'typescript-char';
+import { BraceCounter } from '../language/braceCounter';
+import { TextBuilder } from '../language/textBuilder';
+import { Tokenizer } from '../language/tokenizer';
+import { ITextRangeCollection, IToken, TokenType } from '../language/types';
+
+export class LineFormatter {
+ private builder: TextBuilder;
+ private tokens: ITextRangeCollection<IToken>;
+ private braceCounter: BraceCounter;
+ private text: string;
+
+ // tslint:disable-next-line:cyclomatic-complexity
+ public formatLine(text: string): string {
+ this.tokens = new Tokenizer().tokenize(text);
+ this.text = text;
+ this.builder = new TextBuilder();
+ this.braceCounter = new BraceCounter();
+
+ if (this.tokens.count === 0) {
+ return this.text;
+ }
+
+ const ws = this.text.substr(0, this.tokens.getItemAt(0).start);
+ if (ws.length > 0) {
+ this.builder.append(ws); // Preserve leading indentation
+ }
+
+ for (let i = 0; i < this.tokens.count; i += 1) {
+ const t = this.tokens.getItemAt(i);
+ const prev = i > 0 ? this.tokens.getItemAt(i - 1) : undefined;
+ const next = i < this.tokens.count - 1 ? this.tokens.getItemAt(i + 1) : undefined;
+
+ switch (t.type) {
+ case TokenType.Operator:
+ this.handleOperator(i);
+ break;
+
+ case TokenType.Comma:
+ this.builder.append(',');
+ if (next && !this.isCloseBraceType(next.type)) {
+ this.builder.softAppendSpace();
+ }
+ break;
+
+ case TokenType.Identifier:
+ if (!prev || (!this.isOpenBraceType(prev.type) && prev.type !== TokenType.Colon)) {
+ this.builder.softAppendSpace();
+ }
+ this.builder.append(this.text.substring(t.start, t.end));
+ break;
+
+ case TokenType.Colon:
+ // x: 1 if not in slice, x[1:y] if inside the slice
+ this.builder.append(':');
+ if (!this.braceCounter.isOpened(TokenType.OpenBracket) && (next && next.type !== TokenType.Colon)) {
+ // Not inside opened [[ ... ] sequence
+ this.builder.softAppendSpace();
+ }
+ break;
+
+ case TokenType.Comment:
+ // add space before in-line comment
+ if (prev) {
+ this.builder.softAppendSpace();
+ }
+ this.builder.append(this.text.substring(t.start, t.end));
+ break;
+
+ default:
+ this.handleOther(t);
+ break;
+ }
+ }
+ return this.builder.getText();
+ }
+
+ private handleOperator(index: number): void {
+ const t = this.tokens.getItemAt(index);
+ if (index >= 2 && t.length === 1 && this.text.charCodeAt(t.start) === Char.Equal) {
+ if (this.braceCounter.isOpened(TokenType.OpenBrace)) {
+ // Check if this is = in function arguments. If so, do not
+ // add spaces around it.
+ const prev = this.tokens.getItemAt(index - 1);
+ const prevPrev = this.tokens.getItemAt(index - 2);
+ if (prev.type === TokenType.Identifier &&
+ (prevPrev.type === TokenType.Comma || prevPrev.type === TokenType.OpenBrace)) {
+ this.builder.append('=');
+ return;
+ }
+ }
+ }
+ this.builder.softAppendSpace();
+ this.builder.append(this.text.substring(t.start, t.end));
+ this.builder.softAppendSpace();
+ }
+
+ private handleOther(t: IToken): void {
+ if (this.isBraceType(t.type)) {
+ this.braceCounter.countBrace(t);
+ }
+ this.builder.append(this.text.substring(t.start, t.end));
+ }
+
+ private isOpenBraceType(type: TokenType): boolean {
+ return type === TokenType.OpenBrace || type === TokenType.OpenBracket || type === TokenType.OpenCurly;
+ }
+ private isCloseBraceType(type: TokenType): boolean {
+ return type === TokenType.CloseBrace || type === TokenType.CloseBracket || type === TokenType.CloseCurly;
+ }
+ private isBraceType(type: TokenType): boolean {
+ return this.isOpenBraceType(type) || this.isCloseBraceType(type);
+ }
+}

src/client/language/braceCounter.ts

@@ -0,0 +1,71 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import { IToken, TokenType } from './types';
+
+class BracePair {
+ public readonly openBrace: TokenType;
+ public readonly closeBrace: TokenType;
+
+ constructor(openBrace: TokenType, closeBrace: TokenType) {
+ this.openBrace = openBrace;
+ this.closeBrace = closeBrace;
+ }
+}
+
+class Stack {
+ private store: IToken[] = [];
+ public push(val: IToken) {
+ this.store.push(val);
+ }
+ public pop(): IToken | undefined {
+ return this.store.pop();
+ }
+ public get length(): number {
+ return this.store.length;
+ }
+}
+
+export class BraceCounter {
+ private readonly bracePairs: BracePair[] = [
+ new BracePair(TokenType.OpenBrace, TokenType.CloseBrace),
+ new BracePair(TokenType.OpenBracket, TokenType.CloseBracket),
+ new BracePair(TokenType.OpenCurly, TokenType.CloseCurly)
+ ];
+ private braceStacks: Stack[] = [new Stack(), new Stack(), new Stack()];
+
+ public get count(): number {
+ let c = 0;
+ for (const s of this.braceStacks) {
+ c += s.length;
+ }
+ return c;
+ }
+
+ public isOpened(type: TokenType): boolean {
+ for (let i = 0; i < this.bracePairs.length; i += 1) {
+ const pair = this.bracePairs[i];
+ if (pair.openBrace === type || pair.closeBrace === type) {
+ return this.braceStacks[i].length > 0;
+ }
+ }
+ return false;
+ }
+
+ public countBrace(brace: IToken): boolean {
+ for (let i = 0; i < this.bracePairs.length; i += 1) {
+ const pair = this.bracePairs[i];
+ if (pair.openBrace === brace.type) {
+ this.braceStacks[i].push(brace);
+ return true;
+ }
+ if (pair.closeBrace === brace.type) {
+ if (this.braceStacks[i].length > 0) {
+ this.braceStacks[i].pop();
+ }
+ return true;
+ }
+ }
+ return false;
+ }
+}

src/client/language/characterStream.ts

@@ -4,6 +4,7 @@
 
 // tslint:disable-next-line:import-name
 import Char from 'typescript-char';
+import { isLineBreak, isWhiteSpace } from './characters';
 import { TextIterator } from './textIterator';
 import { ICharacterStream, ITextIterator } from './types';
 
@@ -70,11 +71,11 @@ export class CharacterStream implements ICharacterStream {
  }
 
  public isAtWhiteSpace(): boolean {
- return this.currentChar <= Char.Space || this.currentChar === 0x200B; // Unicode whitespace
+ return isWhiteSpace(this.currentChar);
  }
 
  public isAtLineBreak(): boolean {
- return this.currentChar === Char.CarriageReturn || this.currentChar === Char.LineFeed;
+ return isLineBreak(this.currentChar);
  }
 
  public skipLineBreak(): void {

src/client/language/characters.ts

@@ -0,0 +1,100 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// tslint:disable-next-line:import-name
+import Char from 'typescript-char';
+import { getUnicodeCategory, UnicodeCategory } from './unicode';
+
+export function isIdentifierStartChar(ch: number) {
+ switch (ch) {
+ // Underscore is explicitly allowed to start an identifier
+ case Char.Underscore:
+ return true;
+ // Characters with the Other_ID_Start property
+ case 0x1885:
+ case 0x1886:
+ case 0x2118:
+ case 0x212E:
+ case 0x309B:
+ case 0x309C:
+ return true;
+ default:
+ break;
+ }
+
+ const cat = getUnicodeCategory(ch);
+ switch (cat) {
+ // Supported categories for starting an identifier
+ case UnicodeCategory.UppercaseLetter:
+ case UnicodeCategory.LowercaseLetter:
+ case UnicodeCategory.TitlecaseLetter:
+ case UnicodeCategory.ModifierLetter:
+ case UnicodeCategory.OtherLetter:
+ case UnicodeCategory.LetterNumber:
+ return true;
+ default:
+ break;
+ }
+ return false;
+}
+
+export function isIdentifierChar(ch: number) {
+ if (isIdentifierStartChar(ch)) {
+ return true;
+ }
+
+ switch (ch) {
+ // Characters with the Other_ID_Continue property
+ case 0x00B7:
+ case 0x0387:
+ case 0x1369:
+ case 0x136A:
+ case 0x136B:
+ case 0x136C:
+ case 0x136D:
+ case 0x136E:
+ case 0x136F:
+ case 0x1370:
+ case 0x1371:
+ case 0x19DA:
+ return true;
+ default:
+ break;
+ }
+
+ switch (getUnicodeCategory(ch)) {
+ // Supported categories for continuing an identifier
+ case UnicodeCategory.NonSpacingMark:
+ case UnicodeCategory.SpacingCombiningMark:
+ case UnicodeCategory.DecimalDigitNumber:
+ case UnicodeCategory.ConnectorPunctuation:
+ return true;
+ default:
+ break;
+ }
+ return false;
+}
+
+export function isWhiteSpace(ch: number): boolean {
+ return ch <= Char.Space || ch === 0x200B; // Unicode whitespace
+}
+
+export function isLineBreak(ch: number): boolean {
+ return ch === Char.CarriageReturn || ch === Char.LineFeed;
+}
+
+export function isDecimal(ch: number): boolean {
+ return ch >= Char._0 && ch <= Char._9;
+}
+
+export function isHex(ch: number): boolean {
+ return isDecimal(ch) || (ch >= Char.a && ch <= Char.f) || (ch >= Char.A && ch <= Char.F);
+}
+
+export function isOctal(ch: number): boolean {
+ return ch >= Char._0 && ch <= Char._7;
+}
+
+export function isBinary(ch: number): boolean {
+ return ch === Char._0 || ch === Char._1;
+}

src/client/language/textBuilder.ts

@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import { isWhiteSpace } from './characters';
+
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+export class TextBuilder {
+ private segments: string[] = [];
+
+ public getText(): string {
+ if (this.isLastWhiteSpace()) {
+ this.segments.pop();
+ }
+ return this.segments.join('');
+ }
+
+ public softAppendSpace(): void {
+ if (!this.isLastWhiteSpace() && this.segments.length > 0) {
+ this.segments.push(' ');
+ }
+ }
+
+ public append(text: string): void {
+ this.segments.push(text);
+ }
+
+ private isLastWhiteSpace(): boolean {
+ return this.segments.length > 0 && this.isWhitespace(this.segments[this.segments.length - 1]);
+ }
+
+ private isWhitespace(s: string): boolean {
+ for (let i = 0; i < s.length; i += 1) {
+ if (!isWhiteSpace(s.charCodeAt(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+}