From 415d6a30bfffe6c6d7c419aab1ce6b786b607e9f Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 22 Aug 2023 15:46:03 +0530 Subject: [PATCH] Markdown-It-Py version upgrade (#1835) --- funnel/utils/markdown/base.py | 5 ++--- funnel/utils/markdown/mdit_plugins/abbr.py | 4 ++-- funnel/utils/markdown/mdit_plugins/embeds.py | 4 ++-- .../mdit_plugins/heading_anchors_fix.py | 4 ++-- funnel/utils/markdown/mdit_plugins/ins_tag.py | 18 ++++++++---------- funnel/utils/markdown/mdit_plugins/mark_tag.py | 18 ++++++++---------- funnel/utils/markdown/mdit_plugins/sub_tag.py | 12 ++++++------ funnel/utils/markdown/mdit_plugins/sup_tag.py | 12 ++++++------ funnel/utils/markdown/mdit_plugins/toc.py | 7 ++++--- requirements/base.in | 2 +- requirements/base.txt | 4 ++-- tests/unit/utils/markdown/data/abbr.toml | 4 ++-- 12 files changed, 45 insertions(+), 49 deletions(-) diff --git a/funnel/utils/markdown/base.py b/funnel/utils/markdown/base.py index 8dae459e2..4d7ffb389 100644 --- a/funnel/utils/markdown/base.py +++ b/funnel/utils/markdown/base.py @@ -9,6 +9,7 @@ ClassVar, Dict, Iterable, + Mapping, Optional, Set, Union, @@ -49,8 +50,6 @@ # --- Markdown dataclasses ------------------------------------------------------------- -OptionStrings = Literal['html', 'breaks', 'linkify', 'typographer'] - @dataclass class MarkdownPlugin: @@ -89,7 +88,7 @@ class MarkdownConfig: 'default', 'zero', 'commonmark', 'js-default', 'gfm-like' ] = 'commonmark' #: Updated options against the preset - options_update: Optional[Dict[OptionStrings, bool]] = None + options_update: Optional[Mapping] = None #: Allow only inline rules (skips all block rules)? inline: bool = False diff --git a/funnel/utils/markdown/mdit_plugins/abbr.py b/funnel/utils/markdown/mdit_plugins/abbr.py index aabec1a42..49a81a161 100644 --- a/funnel/utils/markdown/mdit_plugins/abbr.py +++ b/funnel/utils/markdown/mdit_plugins/abbr.py @@ -10,7 +10,7 @@ from markdown_it import MarkdownIt from markdown_it.rules_block import StateBlock -from markdown_it.rules_inline import StateInline +from markdown_it.rules_core import StateCore from markdown_it.token import Token __all__ = ['abbr_plugin'] @@ -57,7 +57,7 @@ def abbr_def(state: StateBlock, start_line: int, end_line: int, silent: bool) -> return True -def abbr_replace(state: StateInline) -> None: +def abbr_replace(state: StateCore) -> None: """Tokenizes and tags defined abbreviations in content.""" block_tokens = state.tokens diff --git a/funnel/utils/markdown/mdit_plugins/embeds.py b/funnel/utils/markdown/mdit_plugins/embeds.py index ea3e47b04..95de85600 100644 --- a/funnel/utils/markdown/mdit_plugins/embeds.py +++ b/funnel/utils/markdown/mdit_plugins/embeds.py @@ -67,7 +67,7 @@ def embeds_func( # Check out the first character quickly, # this should filter out most of non-containers - if marker_char != state.srcCharCode[start]: + if marker_char != ord(state.src[start]): return False # Check out the rest of the marker string @@ -115,7 +115,7 @@ def embeds_func( # test break - if marker_char != state.srcCharCode[start]: + if marker_char != ord(state.src[start]): continue if state.sCount[next_line] - state.blkIndent >= 4: diff --git a/funnel/utils/markdown/mdit_plugins/heading_anchors_fix.py b/funnel/utils/markdown/mdit_plugins/heading_anchors_fix.py index 1039a46d7..be19ed338 100644 --- a/funnel/utils/markdown/mdit_plugins/heading_anchors_fix.py +++ b/funnel/utils/markdown/mdit_plugins/heading_anchors_fix.py @@ -3,12 +3,12 @@ from __future__ import annotations from markdown_it import MarkdownIt -from markdown_it.rules_inline import StateInline +from markdown_it.rules_core import StateCore __all__ = ['heading_anchors_fix_plugin'] -def heading_anchors_fix(state: StateInline) -> None: +def heading_anchors_fix(state: StateCore) -> None: prev_token = None for token in state.tokens: diff --git a/funnel/utils/markdown/mdit_plugins/ins_tag.py b/funnel/utils/markdown/mdit_plugins/ins_tag.py index 8a71fb63c..cc6615ce2 100644 --- a/funnel/utils/markdown/mdit_plugins/ins_tag.py +++ b/funnel/utils/markdown/mdit_plugins/ins_tag.py @@ -17,19 +17,18 @@ __all__ = ['ins_plugin'] -PLUS_CHAR = 0x2B # ASCII value for `+` +PLUS_CHAR = '+' def tokenize(state: StateInline, silent: bool) -> bool: """Insert each marker as a separate text token, and add it to delimiter list.""" start = state.pos - marker = state.srcCharCode[start] - ch = chr(marker) + ch = state.src[start] if silent: return False - if marker != PLUS_CHAR: + if ch != PLUS_CHAR: return False scanned = state.scanDelims(state.pos, True) @@ -50,9 +49,8 @@ def tokenize(state: StateInline, silent: bool) -> bool: token.content = ch + ch state.delimiters.append( Delimiter( - marker=marker, + marker=ord(ch), length=0, # disable "rule of 3" length checks meant for emphasis - jump=i // 2, # for `++` 1 marker = 2 characters token=len(state.tokens) - 1, end=-1, open=scanned.can_open, @@ -71,7 +69,7 @@ def _post_process(state: StateInline, delimiters: List[Delimiter]) -> None: for i in range(0, maximum): start_delim = delimiters[i] - if start_delim.marker != PLUS_CHAR: + if start_delim.marker != ord(PLUS_CHAR): i += 1 continue @@ -85,19 +83,19 @@ def _post_process(state: StateInline, delimiters: List[Delimiter]) -> None: token.type = 'ins_open' token.tag = 'ins' token.nesting = 1 - token.markup = '++' + token.markup = PLUS_CHAR * 2 token.content = '' token = state.tokens[end_delim.token] token.type = 'ins_close' token.tag = 'ins' token.nesting = -1 - token.markup = '++' + token.markup = PLUS_CHAR * 2 token.content = '' end_token = state.tokens[end_delim.token - 1] - if end_token.type == 'text' and end_token == '+': # nosec + if end_token.type == 'text' and end_token == PLUS_CHAR: # nosec lone_markers.append(end_delim.token - 1) # If a marker sequence has an odd number of characters, it's split diff --git a/funnel/utils/markdown/mdit_plugins/mark_tag.py b/funnel/utils/markdown/mdit_plugins/mark_tag.py index 876e1f0d7..17f4939f6 100644 --- a/funnel/utils/markdown/mdit_plugins/mark_tag.py +++ b/funnel/utils/markdown/mdit_plugins/mark_tag.py @@ -17,19 +17,18 @@ __all__ = ['mark_plugin'] -EQUALS_CHAR = 0x3D # ASCII value for `=` +EQUALS_CHAR = '=' def tokenize(state: StateInline, silent: bool) -> bool: """Insert each marker as a separate text token, and add it to delimiter list.""" start = state.pos - marker = state.srcCharCode[start] - ch = chr(marker) + ch = state.src[start] if silent: return False - if marker != EQUALS_CHAR: + if ch != EQUALS_CHAR: return False scanned = state.scanDelims(state.pos, True) @@ -50,9 +49,8 @@ def tokenize(state: StateInline, silent: bool) -> bool: token.content = ch + ch state.delimiters.append( Delimiter( - marker=marker, + marker=ord(ch), length=0, # disable "rule of 3" length checks meant for emphasis - jump=i // 2, # for `==` 1 marker = 2 characters token=len(state.tokens) - 1, end=-1, open=scanned.can_open, @@ -71,7 +69,7 @@ def _post_process(state: StateInline, delimiters: List[Delimiter]) -> None: for i in range(0, maximum): start_delim = delimiters[i] - if start_delim.marker != EQUALS_CHAR: + if start_delim.marker != ord(EQUALS_CHAR): i += 1 continue @@ -85,19 +83,19 @@ def _post_process(state: StateInline, delimiters: List[Delimiter]) -> None: token.type = 'mark_open' token.tag = 'mark' token.nesting = 1 - token.markup = '==' + token.markup = EQUALS_CHAR * 2 token.content = '' token = state.tokens[end_delim.token] token.type = 'mark_close' token.tag = 'mark' token.nesting = -1 - token.markup = '==' + token.markup = EQUALS_CHAR * 2 token.content = '' end_token = state.tokens[end_delim.token - 1] - if end_token.type == 'text' and end_token == '=': # nosec + if end_token.type == 'text' and end_token == EQUALS_CHAR: # nosec lone_markers.append(end_delim.token - 1) # If a marker sequence has an odd number of characters, it's split diff --git a/funnel/utils/markdown/mdit_plugins/sub_tag.py b/funnel/utils/markdown/mdit_plugins/sub_tag.py index 5664f4549..c70823eeb 100644 --- a/funnel/utils/markdown/mdit_plugins/sub_tag.py +++ b/funnel/utils/markdown/mdit_plugins/sub_tag.py @@ -17,7 +17,7 @@ __all__ = ['sub_plugin'] -TILDE_CHAR = 0x7E # ASCII value for `~` +TILDE_CHAR = '~' WHITESPACE_RE = re.compile(r'(^|[^\\])(\\\\)*\s') UNESCAPE_RE = re.compile(r'\\([ \\!"#$%&\'()*+,.\/:;<=>?@[\]^_`{|}~-])') @@ -25,14 +25,14 @@ def tokenize(state: StateInline, silent: bool) -> bool: start = state.pos - marker = state.srcCharCode[start] + ch = state.src[start] maximum = state.posMax found = False if silent: return False - if marker != TILDE_CHAR: + if ch != TILDE_CHAR: return False # Don't run any pairs in validation mode @@ -42,7 +42,7 @@ def tokenize(state: StateInline, silent: bool) -> bool: state.pos = start + 1 while state.pos < maximum: - if state.srcCharCode[state.pos] == TILDE_CHAR: + if state.src[state.pos] == TILDE_CHAR: found = True break state.md.inline.skipToken(state) @@ -63,13 +63,13 @@ def tokenize(state: StateInline, silent: bool) -> bool: # Earlier we checked "not silent", but this implementation does not need it token = state.push('sub_open', 'sub', 1) - token.markup = '~' + token.markup = TILDE_CHAR token = state.push('text', '', 0) token.content = UNESCAPE_RE.sub('$1', content) token = state.push('sub_close', 'sub', -1) - token.markup = '~' + token.markup = TILDE_CHAR state.pos = state.posMax + 1 state.posMax = maximum diff --git a/funnel/utils/markdown/mdit_plugins/sup_tag.py b/funnel/utils/markdown/mdit_plugins/sup_tag.py index 43c0e2029..767892115 100644 --- a/funnel/utils/markdown/mdit_plugins/sup_tag.py +++ b/funnel/utils/markdown/mdit_plugins/sup_tag.py @@ -17,7 +17,7 @@ __all__ = ['sup_plugin'] -CARET_CHAR = 0x5E # ASCII value for `^` +CARET_CHAR = '^' WHITESPACE_RE = re.compile(r'(^|[^\\])(\\\\)*\s') UNESCAPE_RE = re.compile(r'\\([ \\!"#$%&\'()*+,.\/:;<=>?@[\]^_`{|}~-])') @@ -25,14 +25,14 @@ def tokenize(state: StateInline, silent: bool) -> bool: start = state.pos - marker = state.srcCharCode[start] + ch = state.src[start] maximum = state.posMax found = False if silent: return False - if marker != CARET_CHAR: + if ch != CARET_CHAR: return False # Don't run any pairs in validation mode @@ -42,7 +42,7 @@ def tokenize(state: StateInline, silent: bool) -> bool: state.pos = start + 1 while state.pos < maximum: - if state.srcCharCode[state.pos] == CARET_CHAR: + if state.src[state.pos] == CARET_CHAR: found = True break state.md.inline.skipToken(state) @@ -63,13 +63,13 @@ def tokenize(state: StateInline, silent: bool) -> bool: # Earlier we checked "not silent", but this implementation does not need it token = state.push('sup_open', 'sup', 1) - token.markup = '^' + token.markup = CARET_CHAR token = state.push('text', '', 0) token.content = UNESCAPE_RE.sub('$1', content) token = state.push('sup_close', 'sup', -1) - token.markup = '^' + token.markup = CARET_CHAR state.pos = state.posMax + 1 state.posMax = maximum diff --git a/funnel/utils/markdown/mdit_plugins/toc.py b/funnel/utils/markdown/mdit_plugins/toc.py index be9edae94..8d7422822 100644 --- a/funnel/utils/markdown/mdit_plugins/toc.py +++ b/funnel/utils/markdown/mdit_plugins/toc.py @@ -20,6 +20,7 @@ from markdown_it import MarkdownIt from markdown_it.renderer import OptionsDict, RendererHTML +from markdown_it.rules_core import StateCore from markdown_it.rules_inline import StateInline from markdown_it.token import Token @@ -27,7 +28,7 @@ __all__ = ['toc_plugin'] -SQUARE_BRACKET_OPEN_CHAR = 0x5B # ASCII value for `[` +SQUARE_BRACKET_OPEN_CHAR = '[' defaults: Dict = { 'include_level': [1, 2, 3, 4, 5, 6], @@ -197,7 +198,7 @@ def toc_plugin(md: MarkdownIt, **opts) -> None: def toc(state: StateInline, silent: bool) -> bool: # Reject if the token does not start with [ - if state.srcCharCode[state.pos] != SQUARE_BRACKET_OPEN_CHAR: + if state.src[state.pos] != SQUARE_BRACKET_OPEN_CHAR: return False if silent: return False @@ -253,7 +254,7 @@ def toc_body( html = toc_item_to_html(toc, opts, md) return html - def grab_state(state: StateInline): + def grab_state(state: StateCore): state.env['gstate'] = state md.core.ruler.push('grab_state', grab_state) diff --git a/requirements/base.in b/requirements/base.in index 792641774..e268f6359 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -35,7 +35,7 @@ idna ijson itsdangerous linkify-it-py -markdown-it-py<3.0 # Breaks our plugins, needs refactoring +markdown-it-py mdit-py-plugins oauth2 oauth2client diff --git a/requirements/base.txt b/requirements/base.txt index 750fc1b72..eadaed94a 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:5413ec7fabb7788a7e93e2cc0ed7b97ce62aa936 +# SHA1:36d02851efc3c63077a6ffeb9d21fa8db37894d1 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -253,7 +253,7 @@ markdown==3.4.4 # coaster # flask-flatpages # pymdown-extensions -markdown-it-py==2.2.0 +markdown-it-py==3.0.0 # via # -r requirements/base.in # mdit-py-plugins diff --git a/tests/unit/utils/markdown/data/abbr.toml b/tests/unit/utils/markdown/data/abbr.toml index 911616303..94135c96b 100644 --- a/tests/unit/utils/markdown/data/abbr.toml +++ b/tests/unit/utils/markdown/data/abbr.toml @@ -1402,7 +1402,7 @@ document = """

ABBR;
?ABBR?
@ABBR@
-\\ABBR
+\\ABBR
[ABBR[
]ABBR]
ABBR
@@ -2075,7 +2075,7 @@ abbr = """

Testing abbreviations plugin

;ABBR; ?ABBR? @ABBR@ -\\ABBR
+\\ABBR
[ABBR[ ]ABBR] ABBR