Fix parsing of nested f-string specifiers

For an expression like `f"{one:{two:}{three}}"`, `three` is not in an f-string spec, and should be tokenized accordingly.

This PR fixes the `format_spec_count` bookkeeping in the tokenizer, so it properly decrements it when a closing `}` is encountered but only if the `}` closes a format_spec.

Reported in #930.

native/libcst/src/tokenizer/core/mod.rs

@@ -114,12 +114,8 @@ pub enum TokType {
  Dedent,
  Async,
  Await,
- // TODO; add support for these
- #[allow(dead_code)]
  FStringStart,
- #[allow(dead_code)]
  FStringString,
- #[allow(dead_code)]
  FStringEnd,
  EndMarker,
 }

native/libcst/src/tokenizer/core/string_types.rs

@@ -98,11 +98,10 @@ impl FStringNode {
  }
 
  pub fn close_parentheses(&mut self) {
- self.parentheses_count -= 1;
- if self.parentheses_count == 0 {
- // No parentheses means that the format spec is also finished.
- self.format_spec_count = 0;
+ if self.is_in_format_spec() {
+ self.format_spec_count -= 1;
  }
+ self.parentheses_count -= 1;
  }
 
  pub fn allow_multiline(&self) -> bool {

native/libcst/src/tokenizer/tests.rs

@@ -814,3 +814,29 @@ fn test_inconsistent_indentation_at_eof() {
  ])
  )
 }
+
+#[test]
+fn test_nested_f_string_specs() {
+ let config = TokConfig {
+ split_fstring: true,
+ ..default_config()
+ };
+ assert_eq!(
+ tokenize_all("f'{_:{_:}{_}}'", &config),
+ Ok(vec![
+ (TokType::FStringStart, "f'"),
+ (TokType::Op, "{"),
+ (TokType::Name, "_"),
+ (TokType::Op, ":"),
+ (TokType::Op, "{"),
+ (TokType::Name, "_"),
+ (TokType::Op, ":"),
+ (TokType::Op, "}"),
+ (TokType::Op, "{"),
+ (TokType::Name, "_"),
+ (TokType::Op, "}"),
+ (TokType::Op, "}"),
+ (TokType::FStringEnd, "'")
+ ])
+ )
+}

native/libcst/tests/fixtures/super_strings.py

@@ -30,3 +30,5 @@
 f'\{{\}}'
 f"regexp_like(path, '.*\{file_type}$')"
 f"\lfoo"
+
+f"{_:{_:}{a}}"