Markdown-It-Py version upgrade (#1835)

funnel/utils/markdown/base.py

@@ -9,6 +9,7 @@
  ClassVar,
  Dict,
  Iterable,
+ Mapping,
  Optional,
  Set,
  Union,
@@ -49,8 +50,6 @@
 
 # --- Markdown dataclasses -------------------------------------------------------------
 
-OptionStrings = Literal['html', 'breaks', 'linkify', 'typographer']
-
 
 @dataclass
 class MarkdownPlugin:
@@ -89,7 +88,7 @@ class MarkdownConfig:
  'default', 'zero', 'commonmark', 'js-default', 'gfm-like'
  ] = 'commonmark'
  #: Updated options against the preset
- options_update: Optional[Dict[OptionStrings, bool]] = None
+ options_update: Optional[Mapping] = None
  #: Allow only inline rules (skips all block rules)?
  inline: bool = False
 

funnel/utils/markdown/mdit_plugins/abbr.py

@@ -10,7 +10,7 @@
 
 from markdown_it import MarkdownIt
 from markdown_it.rules_block import StateBlock
-from markdown_it.rules_inline import StateInline
+from markdown_it.rules_core import StateCore
 from markdown_it.token import Token
 
 __all__ = ['abbr_plugin']
@@ -57,7 +57,7 @@ def abbr_def(state: StateBlock, start_line: int, end_line: int, silent: bool) ->
  return True
 
 
-def abbr_replace(state: StateInline) -> None:
+def abbr_replace(state: StateCore) -> None:
  """Tokenizes and tags defined abbreviations in content."""
  block_tokens = state.tokens
 

funnel/utils/markdown/mdit_plugins/embeds.py

@@ -67,7 +67,7 @@ def embeds_func(
 
  # Check out the first character quickly,
  # this should filter out most of non-containers
- if marker_char != state.srcCharCode[start]:
+ if marker_char != ord(state.src[start]):
  return False
 
  # Check out the rest of the marker string
@@ -115,7 +115,7 @@ def embeds_func(
  # test
  break
 
- if marker_char != state.srcCharCode[start]:
+ if marker_char != ord(state.src[start]):
  continue
 
  if state.sCount[next_line] - state.blkIndent >= 4:

funnel/utils/markdown/mdit_plugins/heading_anchors_fix.py

@@ -3,12 +3,12 @@
 from __future__ import annotations
 
 from markdown_it import MarkdownIt
-from markdown_it.rules_inline import StateInline
+from markdown_it.rules_core import StateCore
 
 __all__ = ['heading_anchors_fix_plugin']
 
 
-def heading_anchors_fix(state: StateInline) -> None:
+def heading_anchors_fix(state: StateCore) -> None:
  prev_token = None
 
  for token in state.tokens:

funnel/utils/markdown/mdit_plugins/ins_tag.py

@@ -17,19 +17,18 @@
 
 __all__ = ['ins_plugin']
 
-PLUS_CHAR = 0x2B # ASCII value for `+`
+PLUS_CHAR = '+'
 
 
 def tokenize(state: StateInline, silent: bool) -> bool:
  """Insert each marker as a separate text token, and add it to delimiter list."""
  start = state.pos
- marker = state.srcCharCode[start]
- ch = chr(marker)
+ ch = state.src[start]
 
  if silent:
  return False
 
- if marker != PLUS_CHAR:
+ if ch != PLUS_CHAR:
  return False
 
  scanned = state.scanDelims(state.pos, True)
@@ -50,9 +49,8 @@ def tokenize(state: StateInline, silent: bool) -> bool:
  token.content = ch + ch
  state.delimiters.append(
  Delimiter(
- marker=marker,
+ marker=ord(ch),
  length=0, # disable "rule of 3" length checks meant for emphasis
- jump=i // 2, # for `++` 1 marker = 2 characters
  token=len(state.tokens) - 1,
  end=-1,
  open=scanned.can_open,
@@ -71,7 +69,7 @@ def _post_process(state: StateInline, delimiters: List[Delimiter]) -> None:
 
  for i in range(0, maximum):
  start_delim = delimiters[i]
- if start_delim.marker != PLUS_CHAR:
+ if start_delim.marker != ord(PLUS_CHAR):
  i += 1
  continue
 
@@ -85,19 +83,19 @@ def _post_process(state: StateInline, delimiters: List[Delimiter]) -> None:
  token.type = 'ins_open'
  token.tag = 'ins'
  token.nesting = 1
- token.markup = '++'
+ token.markup = PLUS_CHAR * 2
  token.content = ''
 
  token = state.tokens[end_delim.token]
  token.type = 'ins_close'
  token.tag = 'ins'
  token.nesting = -1
- token.markup = '++'
+ token.markup = PLUS_CHAR * 2
  token.content = ''
 
  end_token = state.tokens[end_delim.token - 1]
 
- if end_token.type == 'text' and end_token == '+': # nosec
+ if end_token.type == 'text' and end_token == PLUS_CHAR: # nosec
  lone_markers.append(end_delim.token - 1)
 
  # If a marker sequence has an odd number of characters, it's split

funnel/utils/markdown/mdit_plugins/mark_tag.py

@@ -17,19 +17,18 @@
 
 __all__ = ['mark_plugin']
 
-EQUALS_CHAR = 0x3D # ASCII value for `=`
+EQUALS_CHAR = '='
 
 
 def tokenize(state: StateInline, silent: bool) -> bool:
  """Insert each marker as a separate text token, and add it to delimiter list."""
  start = state.pos
- marker = state.srcCharCode[start]
- ch = chr(marker)
+ ch = state.src[start]
 
  if silent:
  return False
 
- if marker != EQUALS_CHAR:
+ if ch != EQUALS_CHAR:
  return False
 
  scanned = state.scanDelims(state.pos, True)
@@ -50,9 +49,8 @@ def tokenize(state: StateInline, silent: bool) -> bool:
  token.content = ch + ch
  state.delimiters.append(
  Delimiter(
- marker=marker,
+ marker=ord(ch),
  length=0, # disable "rule of 3" length checks meant for emphasis
- jump=i // 2, # for `==` 1 marker = 2 characters
  token=len(state.tokens) - 1,
  end=-1,
  open=scanned.can_open,
@@ -71,7 +69,7 @@ def _post_process(state: StateInline, delimiters: List[Delimiter]) -> None:
 
  for i in range(0, maximum):
  start_delim = delimiters[i]
- if start_delim.marker != EQUALS_CHAR:
+ if start_delim.marker != ord(EQUALS_CHAR):
  i += 1
  continue
 
@@ -85,19 +83,19 @@ def _post_process(state: StateInline, delimiters: List[Delimiter]) -> None:
  token.type = 'mark_open'
  token.tag = 'mark'
  token.nesting = 1
- token.markup = '=='
+ token.markup = EQUALS_CHAR * 2
  token.content = ''
 
  token = state.tokens[end_delim.token]
  token.type = 'mark_close'
  token.tag = 'mark'
  token.nesting = -1
- token.markup = '=='
+ token.markup = EQUALS_CHAR * 2
  token.content = ''
 
  end_token = state.tokens[end_delim.token - 1]
 
- if end_token.type == 'text' and end_token == '=': # nosec
+ if end_token.type == 'text' and end_token == EQUALS_CHAR: # nosec
  lone_markers.append(end_delim.token - 1)
 
  # If a marker sequence has an odd number of characters, it's split

funnel/utils/markdown/mdit_plugins/sub_tag.py

@@ -17,22 +17,22 @@
 
 __all__ = ['sub_plugin']
 
-TILDE_CHAR = 0x7E # ASCII value for `~`
+TILDE_CHAR = '~'
 
 WHITESPACE_RE = re.compile(r'(^|[^\\])(\\\\)*\s')
 UNESCAPE_RE = re.compile(r'\\([ \\!"#$%&\'()*+,.\/:;<=>?@[\]^_`{|}~-])')
 
 
 def tokenize(state: StateInline, silent: bool) -> bool:
  start = state.pos
- marker = state.srcCharCode[start]
+ ch = state.src[start]
  maximum = state.posMax
  found = False
 
  if silent:
  return False
 
- if marker != TILDE_CHAR:
+ if ch != TILDE_CHAR:
  return False
 
  # Don't run any pairs in validation mode
@@ -42,7 +42,7 @@ def tokenize(state: StateInline, silent: bool) -> bool:
  state.pos = start + 1
 
  while state.pos < maximum:
- if state.srcCharCode[state.pos] == TILDE_CHAR:
+ if state.src[state.pos] == TILDE_CHAR:
  found = True
  break
  state.md.inline.skipToken(state)
@@ -63,13 +63,13 @@ def tokenize(state: StateInline, silent: bool) -> bool:
 
  # Earlier we checked "not silent", but this implementation does not need it
  token = state.push('sub_open', 'sub', 1)
- token.markup = '~'
+ token.markup = TILDE_CHAR
 
  token = state.push('text', '', 0)
  token.content = UNESCAPE_RE.sub('$1', content)
 
  token = state.push('sub_close', 'sub', -1)
- token.markup = '~'
+ token.markup = TILDE_CHAR
 
  state.pos = state.posMax + 1
  state.posMax = maximum

funnel/utils/markdown/mdit_plugins/sup_tag.py

@@ -17,22 +17,22 @@
 
 __all__ = ['sup_plugin']
 
-CARET_CHAR = 0x5E # ASCII value for `^`
+CARET_CHAR = '^'
 
 WHITESPACE_RE = re.compile(r'(^|[^\\])(\\\\)*\s')
 UNESCAPE_RE = re.compile(r'\\([ \\!"#$%&\'()*+,.\/:;<=>?@[\]^_`{|}~-])')
 
 
 def tokenize(state: StateInline, silent: bool) -> bool:
  start = state.pos
- marker = state.srcCharCode[start]
+ ch = state.src[start]
  maximum = state.posMax
  found = False
 
  if silent:
  return False
 
- if marker != CARET_CHAR:
+ if ch != CARET_CHAR:
  return False
 
  # Don't run any pairs in validation mode
@@ -42,7 +42,7 @@ def tokenize(state: StateInline, silent: bool) -> bool:
  state.pos = start + 1
 
  while state.pos < maximum:
- if state.srcCharCode[state.pos] == CARET_CHAR:
+ if state.src[state.pos] == CARET_CHAR:
  found = True
  break
  state.md.inline.skipToken(state)
@@ -63,13 +63,13 @@ def tokenize(state: StateInline, silent: bool) -> bool:
 
  # Earlier we checked "not silent", but this implementation does not need it
  token = state.push('sup_open', 'sup', 1)
- token.markup = '^'
+ token.markup = CARET_CHAR
 
  token = state.push('text', '', 0)
  token.content = UNESCAPE_RE.sub('$1', content)
 
  token = state.push('sup_close', 'sup', -1)
- token.markup = '^'
+ token.markup = CARET_CHAR
 
  state.pos = state.posMax + 1
  state.posMax = maximum

funnel/utils/markdown/mdit_plugins/toc.py

@@ -20,14 +20,15 @@
 
 from markdown_it import MarkdownIt
 from markdown_it.renderer import OptionsDict, RendererHTML
+from markdown_it.rules_core import StateCore
 from markdown_it.rules_inline import StateInline
 from markdown_it.token import Token
 
 from coaster.utils import make_name
 
 __all__ = ['toc_plugin']
 
-SQUARE_BRACKET_OPEN_CHAR = 0x5B # ASCII value for `[`
+SQUARE_BRACKET_OPEN_CHAR = '['
 
 defaults: Dict = {
  'include_level': [1, 2, 3, 4, 5, 6],
@@ -197,7 +198,7 @@ def toc_plugin(md: MarkdownIt, **opts) -> None:
 
  def toc(state: StateInline, silent: bool) -> bool:
  # Reject if the token does not start with [
- if state.srcCharCode[state.pos] != SQUARE_BRACKET_OPEN_CHAR:
+ if state.src[state.pos] != SQUARE_BRACKET_OPEN_CHAR:
  return False
  if silent:
  return False
@@ -253,7 +254,7 @@ def toc_body(
  html = toc_item_to_html(toc, opts, md)
  return html
 
- def grab_state(state: StateInline):
+ def grab_state(state: StateCore):
  state.env['gstate'] = state
 
  md.core.ruler.push('grab_state', grab_state)

requirements/base.in

@@ -35,7 +35,7 @@ idna
 ijson
 itsdangerous
 linkify-it-py
-markdown-it-py<3.0 # Breaks our plugins, needs refactoring
+markdown-it-py
 mdit-py-plugins
 oauth2
 oauth2client

requirements/base.txt

@@ -1,4 +1,4 @@
-# SHA1:5413ec7fabb7788a7e93e2cc0ed7b97ce62aa936
+# SHA1:36d02851efc3c63077a6ffeb9d21fa8db37894d1
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -253,7 +253,7 @@ markdown==3.4.4
  # coaster
  # flask-flatpages
  # pymdown-extensions
-markdown-it-py==2.2.0
+markdown-it-py==3.0.0
  # via
  # -r requirements/base.in
  # mdit-py-plugins

tests/unit/utils/markdown/data/abbr.toml

@@ -1402,7 +1402,7 @@ document = """<h2 id="h:testing-abbreviations-plugin"><a href="#h:testing-abbrev
 ;<abbr title="Abbreviation">ABBR</abbr>;<br />
 ?<abbr title="Abbreviation">ABBR</abbr>?<br />
 @<abbr title="Abbreviation">ABBR</abbr>@<br />
-\\<abbr title="Abbreviation">ABBR</abbr><br />
+\\ABBR<br />
 [<abbr title="Abbreviation">ABBR</abbr>[<br />
 ]<abbr title="Abbreviation">ABBR</abbr>]<br />
 <em><abbr title="Abbreviation">ABBR</abbr></em><br />
@@ -2075,7 +2075,7 @@ abbr = """<h2>Testing abbreviations plugin</h2>
 ;<abbr title="Abbreviation">ABBR</abbr>;
 ?<abbr title="Abbreviation">ABBR</abbr>?
 @<abbr title="Abbreviation">ABBR</abbr>@
-\\<abbr title="Abbreviation">ABBR</abbr><br>
+\\ABBR<br>
 [<abbr title="Abbreviation">ABBR</abbr>[
 ]<abbr title="Abbreviation">ABBR</abbr>]
 <em><abbr title="Abbreviation">ABBR</abbr></em>