From 5de41fb8e527004dbc363e047a3c380c9288c74f Mon Sep 17 00:00:00 2001 From: Hsiaoming Yang Date: Sun, 21 Jun 2026 18:29:09 +0900 Subject: [PATCH] refactor: refactor on parser, support all commonmark rules --- benchmark/bench.py | 4 +- docs/index.rst | 2 +- docs/plugins.rst | 6 + src/mistune/__init__.py | 6 +- src/mistune/__main__.py | 2 +- src/mistune/block_parser.py | 121 ++++-- src/mistune/core.py | 22 +- src/mistune/helpers.py | 134 +++++-- src/mistune/inline_parser.py | 550 +++++++++++++++++++++++---- src/mistune/list_parser.py | 289 +++++++++----- src/mistune/plugins/def_list.py | 156 +++++--- src/mistune/plugins/speedup.py | 48 +-- src/mistune/plugins/spoiler.py | 4 +- src/mistune/plugins/table.py | 120 ++++-- src/mistune/util.py | 2 + tests/fixtures/diff-commonmark.txt | 76 ---- tests/fixtures/fenced_admonition.txt | 14 +- tests/fixtures/fix-commonmark.txt | 52 +-- tests/fixtures/renderer_markdown.txt | 2 +- tests/test_commonmark.py | 58 +-- tests/test_misc.py | 75 ++++ tests/test_syntax.py | 1 - 22 files changed, 1199 insertions(+), 545 deletions(-) delete mode 100644 tests/fixtures/diff-commonmark.txt Index: mistune-3.1.3/benchmark/bench.py =================================================================== --- mistune-3.1.3.orig/benchmark/bench.py +++ mistune-3.1.3/benchmark/bench.py @@ -48,8 +48,7 @@ def get_markdown_parsers(): ) parsers[f"mistune ({mistune.__version__})"] = mistune.html - parsers[f"mistune (slow)"] = mistune.create_markdown(escape=False) - parsers[f"mistune (fast)"] = mistune.create_markdown(escape=False, plugins=["speedup"]) + parsers["mistune (core)"] = mistune.create_markdown(escape=False) parsers["mistune (full)"] = mistune.create_markdown( escape=False, plugins=[ @@ -73,7 +72,6 @@ def get_markdown_parsers(): Include(), ] ), - "speedup", ], ) Index: mistune-3.1.3/docs/index.rst =================================================================== --- mistune-3.1.3.orig/docs/index.rst +++ mistune-3.1.3/docs/index.rst @@ -9,7 +9,7 @@ Mistune: Python Markdown Parser Release v\ |version|. A fast yet powerful Python Markdown parser with renderers and plugins, -compatible with sane CommonMark rules. +compatible with CommonMark 0.31.2. Using old Mistune? Checkout docs: Index: mistune-3.1.3/docs/plugins.rst =================================================================== --- mistune-3.1.3.orig/docs/plugins.rst +++ mistune-3.1.3/docs/plugins.rst @@ -8,6 +8,12 @@ Built-in Plugins Mistune offers many built-in plugins, including all the popular markups. +.. note:: + + The historical ``speedup`` plugin is kept only for compatibility. Its + paragraph and inline text fast paths are now built into the core parsers, so + ``plugins=['speedup']`` is accepted but ignored. + .. _strikethrough: strikethrough Index: mistune-3.1.3/src/mistune/__init__.py =================================================================== --- mistune-3.1.3.orig/src/mistune/__init__.py +++ mistune-3.1.3/src/mistune/__init__.py @@ -3,7 +3,7 @@ mistune ~~~~~~~ A fast yet powerful Python Markdown parser with renderers and -plugins, compatible with sane CommonMark rules. +plugins, compatible with CommonMark 0.31.2. Documentation: https://mistune.lepture.com/ """ @@ -51,11 +51,11 @@ def create_markdown( inline = InlineParser(hard_wrap=hard_wrap) real_plugins: Optional[Iterable[Plugin]] = None if plugins is not None: - real_plugins = [import_plugin(n) for n in plugins] + real_plugins = [import_plugin(n) for n in plugins if n != "speedup"] return Markdown(renderer=renderer, inline=inline, plugins=real_plugins) -html: Markdown = create_markdown(escape=False, plugins=["strikethrough", "footnotes", "table", "speedup"]) +html: Markdown = create_markdown(escape=False, plugins=["strikethrough", "footnotes", "table"]) __cached_parsers: Dict[Tuple[bool, Optional[RendererRef], Optional[Iterable[Any]]], Markdown] = {} Index: mistune-3.1.3/src/mistune/__main__.py =================================================================== --- mistune-3.1.3.orig/src/mistune/__main__.py +++ mistune-3.1.3/src/mistune/__main__.py @@ -17,7 +17,7 @@ def _md(args: argparse.Namespace) -> "Ma plugins = args.plugin else: # default plugins - plugins = ["strikethrough", "footnotes", "table", "speedup"] + plugins = ["strikethrough", "footnotes", "table"] if args.renderer == "rst": renderer: "BaseRenderer" = RSTRenderer() Index: mistune-3.1.3/src/mistune/block_parser.py =================================================================== --- mistune-3.1.3.orig/src/mistune/block_parser.py +++ mistune-3.1.3/src/mistune/block_parser.py @@ -1,5 +1,6 @@ import re -from typing import Optional, List, Tuple, Match, Pattern +from typing import Optional, List, Tuple, Match, Pattern, Set +import string from .util import ( unikey, escape_url, @@ -22,15 +23,13 @@ from .list_parser import parse_list, LIS _INDENT_CODE_TRIM = re.compile(r"^ {1,4}", flags=re.M) _ATX_HEADING_TRIM = re.compile(r"(\s+|^)#+\s*$") _BLOCK_QUOTE_TRIM = re.compile(r"^ ?", flags=re.M) -_BLOCK_QUOTE_LEADING = re.compile(r"^ *>", flags=re.M) -_LINE_BLANK_END = re.compile(r"\n[ \t]*\n$") _BLANK_TO_LINE = re.compile(r"[ \t]*\n") _BLOCK_TAGS_PATTERN = "(" + "|".join(BLOCK_TAGS) + "|" + "|".join(PRE_TAGS) + ")" _OPEN_TAG_END = re.compile(HTML_ATTRIBUTES + r"[ \t]*>[ \t]*(?:\n|$)") _CLOSE_TAG_END = re.compile(r"[ \t]*>[ \t]*(?:\n|$)") -_STRICT_BLOCK_QUOTE = re.compile(r"( {0,3}>[^\n]*(?:\n|$))+") +_BLOCK_QUOTE_LINE = re.compile(r"^ {0,3}>([^\n]*(?:\n|$))") class BlockParser(Parser[BlockState]): @@ -93,7 +92,7 @@ class BlockParser(Parser[BlockState]): self, block_quote_rules: Optional[List[str]] = None, list_rules: Optional[List[str]] = None, - max_nested_level: int = 6, + max_nested_level: int = 100, ): super(BlockParser, self).__init__() @@ -128,11 +127,14 @@ class BlockParser(Parser[BlockState]): return end_pos code = m.group(0) + end_pos = _trim_partial_next_line_indent(code, m.end()) + if end_pos != m.end(): + code = state.get_text(end_pos) code = expand_leading_tab(code) code = _INDENT_CODE_TRIM.sub("", code) code = code.strip("\n") state.append_token({"type": "block_code", "raw": code, "style": "indent"}) - return m.end() + return end_pos def parse_fenced_code(self, m: Match[str], state: BlockState) -> Optional[int]: """Parse token for fenced code block. A fenced code block is started with @@ -202,6 +204,9 @@ class BlockParser(Parser[BlockState]): H1 title ======== """ + if state.cursor in state.lazy_line_starts: + return None + last_token = state.last_token() if last_token and last_token["type"] == "paragraph": level = 1 if m.group("setext_1") == "=" else 2 @@ -285,29 +290,26 @@ class BlockParser(Parser[BlockState]): state.env["ref_links"][key] = data return end_pos - def extract_block_quote(self, m: Match[str], state: BlockState) -> Tuple[str, Optional[int]]: + def extract_block_quote(self, m: Match[str], state: BlockState) -> Tuple[str, Optional[int], Set[int]]: """Extract text and cursor end position of a block quote.""" - # cleanup at first to detect if it is code block - text = m.group("quote_1") + "\n" - text = expand_leading_tab(text, 3) - text = _BLOCK_QUOTE_TRIM.sub("", text) + text = _parse_block_quote_line(state.get_line(state.cursor)) + assert text is not None + lazy_line_starts: Set[int] = set() sc = self.compile_sc(["blank_line", "indent_code", "fenced_code"]) require_marker = bool(sc.match(text)) - state.cursor = m.end() + 1 + state.cursor += len(state.get_line(state.cursor)) end_pos: Optional[int] = None if require_marker: - m2 = _STRICT_BLOCK_QUOTE.match(state.src, state.cursor) - if m2: - quote = m2.group(0) - quote = _BLOCK_QUOTE_LEADING.sub("", quote) - quote = expand_leading_tab(quote, 3) - quote = _BLOCK_QUOTE_TRIM.sub("", quote) + while state.cursor < state.cursor_max: + quote = _parse_block_quote_line(state.get_line(state.cursor)) + if quote is None: + break text += quote - state.cursor = m2.end() + state.cursor += len(state.get_line(state.cursor)) else: prev_blank_line = False break_sc = self.compile_sc( @@ -320,18 +322,14 @@ class BlockParser(Parser[BlockState]): ] ) while state.cursor < state.cursor_max: - m3 = _STRICT_BLOCK_QUOTE.match(state.src, state.cursor) - if m3: - quote = m3.group(0) - quote = _BLOCK_QUOTE_LEADING.sub("", quote) - quote = expand_leading_tab(quote, 3) - quote = _BLOCK_QUOTE_TRIM.sub("", quote) + quote = _parse_block_quote_line(state.get_line(state.cursor)) + if quote is not None: text += quote - state.cursor = m3.end() + state.cursor += len(state.get_line(state.cursor)) if not quote.strip(): prev_blank_line = True else: - prev_blank_line = bool(_LINE_BLANK_END.search(quote)) + prev_blank_line = False continue if prev_blank_line: @@ -347,15 +345,14 @@ class BlockParser(Parser[BlockState]): break # lazy continuation line - pos = state.find_line_end() - line = state.get_text(pos) - line = expand_leading_tab(line, 3) - text += line - state.cursor = pos + line = state.get_line(state.cursor) + lazy_line_starts.add(len(text)) + text += expand_leading_tab(line, 3) + state.cursor += len(line) # according to CommonMark Example 6, the second tab should be # treated as 4 spaces - return expand_tab(text), end_pos + return expand_tab(text), end_pos, lazy_line_starts def parse_block_quote(self, m: Match[str], state: BlockState) -> int: """Parse token for block quote. Here is an example of the syntax: @@ -365,9 +362,9 @@ class BlockParser(Parser[BlockState]): > a block quote starts > with right arrows """ - text, end_pos = self.extract_block_quote(m, state) + text, end_pos, lazy_line_starts = self.extract_block_quote(m, state) # scan children state - child = state.child_state(text) + child = state.child_state(text, lazy_line_starts=lazy_line_starts) if state.depth() >= self.max_nested_level - 1: rules = list(self.block_quote_rules) rules.remove("block_quote") @@ -444,7 +441,12 @@ class BlockParser(Parser[BlockState]): sc = self.compile_sc(rules) while state.cursor < state.cursor_max: - m = sc.search(state.src, state.cursor) + m = sc.match(state.src, state.cursor) + if not m and self._parse_plain_paragraph(state, sc): + continue + + if not m: + m = sc.search(state.src, state.cursor) if not m: break @@ -468,6 +470,28 @@ class BlockParser(Parser[BlockState]): state.add_paragraph(text) state.cursor = state.cursor_max + def _parse_plain_paragraph(self, state: BlockState, sc: Pattern[str]) -> bool: + if not _is_plain_paragraph_start(state.src, state.cursor): + return False + + pos = state.cursor + while pos < state.cursor_max: + if pos > state.cursor and sc.match(state.src, pos): + break + + line = state.get_line(pos) + if not line.strip(): + break + + pos += len(line) + + if pos <= state.cursor: + return False + + state.add_paragraph(state.get_text(pos)) + state.cursor = pos + return True + def _parse_html_to_end(state: BlockState, end_marker: str, start_pos: int) -> int: marker_pos = state.src.find(end_marker, start_pos) @@ -495,3 +519,29 @@ def _parse_html_to_newline(state: BlockS state.append_token({"type": "block_html", "raw": text}) return end_pos + + +def _parse_block_quote_line(line: str) -> Optional[str]: + m = _BLOCK_QUOTE_LINE.match(line) + if not m: + return None + text = expand_leading_tab(m.group(1), 3) + return _BLOCK_QUOTE_TRIM.sub("", text) + + +def _trim_partial_next_line_indent(text: str, end_pos: int) -> int: + line_start = text.rfind("\n") + 1 + if line_start == 0: + return end_pos + + suffix = text[line_start:] + if suffix and suffix.strip(" \t") == "" and len(suffix.expandtabs(4)) < 4: + return end_pos - len(suffix) + return end_pos + + +def _is_plain_paragraph_start(src: str, pos: int) -> bool: + if pos >= len(src): + return False + c = src[pos] + return not c.isspace() and not c.isdigit() and c not in string.punctuation Index: mistune-3.1.3/src/mistune/core.py =================================================================== --- mistune-3.1.3.orig/src/mistune/core.py +++ mistune-3.1.3/src/mistune/core.py @@ -12,6 +12,8 @@ from typing import ( MutableMapping, Optional, Pattern, + Set, + Tuple, Type, TypeVar, Union, @@ -36,6 +38,7 @@ class BlockState: list_tight: bool parent: Any env: MutableMapping[str, Any] + lazy_line_starts: Set[int] def __init__(self, parent: Optional[Any] = None) -> None: self.src = "" @@ -48,6 +51,7 @@ class BlockState: # for list and block quote chain self.list_tight = True self.parent = parent + self.lazy_line_starts = set() # for saving def references if parent: @@ -55,9 +59,11 @@ class BlockState: else: self.env = {"ref_links": {}} - def child_state(self, src: str) -> "BlockState": + def child_state(self, src: str, lazy_line_starts: Optional[Set[int]] = None) -> "BlockState": child = self.__class__(self) child.process(src) + if lazy_line_starts: + child.lazy_line_starts = lazy_line_starts return child def process(self, src: str) -> None: @@ -65,13 +71,19 @@ class BlockState: self.cursor_max = len(src) def find_line_end(self) -> int: - m = _LINE_END.search(self.src, self.cursor) + return self.find_line_end_at(self.cursor) + + def find_line_end_at(self, pos: int) -> int: + m = _LINE_END.search(self.src, pos) assert m is not None return m.end() def get_text(self, end_pos: int) -> str: return self.src[self.cursor : end_pos] + def get_line(self, start_pos: int) -> str: + return self.src[start_pos : self.find_line_end_at(start_pos)] + def last_token(self) -> Any: if self.tokens: return self.tokens[-1] @@ -117,9 +129,8 @@ class InlineState: self.tokens: List[Dict[str, Any]] = [] self.in_image = False self.in_link = False - self.in_emphasis = False - self.in_strong = False self.no_close_bracket_before: int = 0 # high-water mark for DoS mitigation + self.link_brackets: Dict[int, Tuple[str, Dict[int, int]]] = {} def prepend_token(self, token: Dict[str, Any]) -> None: """Insert token before the last token.""" @@ -134,8 +145,7 @@ class InlineState: state = self.__class__(self.env) state.in_image = self.in_image state.in_link = self.in_link - state.in_emphasis = self.in_emphasis - state.in_strong = self.in_strong + state.link_brackets = self.link_brackets return state Index: mistune-3.1.3/src/mistune/helpers.py =================================================================== --- mistune-3.1.3.orig/src/mistune/helpers.py +++ mistune-3.1.3/src/mistune/helpers.py @@ -9,21 +9,7 @@ PUNCTUATION = r"[" + re.escape(string.pu LINK_LABEL = r"(?:[^\\\[\]]|\\.){0,500}" -LINK_BRACKET_START = re.compile(r"[ \t]*\n?[ \t]*<") -LINK_BRACKET_RE = re.compile(r"<([^<>\n\\\x00]*)>") -LINK_HREF_BLOCK_RE = re.compile(r"[ \t]*\n?[ \t]*([^\s]+)(?:\s|$)") -LINK_HREF_INLINE_RE = re.compile( - r"[ \t]*\n?[ \t]*([^ \t\n]*?)(?:[ \t\n]|" - r"(?:" + PREVENT_BACKSLASH + r"\)))" -) - -LINK_TITLE_RE = re.compile( - r"[ \t\n]+(" - r'"(?:\\' + PUNCTUATION + r'|[^"\\\x00])*"|' - r"'(?:\\" + PUNCTUATION + r"|[^'\\\x00])*'" - r")" -) -PAREN_END_RE = re.compile(r"\s*\)") +ASCII_WHITESPACE = " \t\n\r\f" HTML_TAGNAME = r"[A-Za-z][A-Za-z0-9-]*" HTML_ATTRIBUTES = ( @@ -142,36 +128,70 @@ def parse_link_label(src: str, start_pos def parse_link_href(src: str, start_pos: int, block: bool = False) -> Union[Tuple[str, int], Tuple[None, None]]: - m = LINK_BRACKET_START.match(src, start_pos) - if m: - start_pos = m.end() - 1 - m = LINK_BRACKET_RE.match(src, start_pos) - if m: - return m.group(1), m.end() + pos = _skip_link_start_whitespace(src, start_pos) + if pos >= len(src): return None, None - if block: - m = LINK_HREF_BLOCK_RE.match(src, start_pos) - else: - m = LINK_HREF_INLINE_RE.match(src, start_pos) - - if not m: + if src[pos] == "<": + return _parse_angle_link_href(src, pos) + if block and src[pos] in ASCII_WHITESPACE: return None, None - end_pos = m.end() - href = m.group(1) + start = pos + level = 0 + while pos < len(src): + c = src[pos] + if c in ASCII_WHITESPACE: + break + if c == "\x00": + return None, None + if c == "\\": + pos = min(pos + 2, len(src)) + continue + if not block: + if c == "(": + level += 1 + elif c == ")": + if level == 0: + break + level -= 1 + pos += 1 - if block and src[end_pos - 1] == href[-1]: - return href, end_pos - return href, end_pos - 1 + if not block and level != 0: + return None, None + return src[start:pos], pos def parse_link_title(src: str, start_pos: int, max_pos: int) -> Union[Tuple[str, int], Tuple[None, None]]: - m = LINK_TITLE_RE.match(src, start_pos, max_pos) - if m: - title = m.group(1)[1:-1] - title = unescape_char(title) - return title, m.end() + pos = start_pos + if pos >= max_pos or src[pos] not in ASCII_WHITESPACE: + return None, None + + pos = _skip_ascii_whitespace(src, pos, max_pos) + if pos >= max_pos: + return None, None + + opener = src[pos] + closer = {"'": "'", '"': '"', "(": ")"}.get(opener) + if closer is None: + return None, None + + pos += 1 + title = [] + while pos < max_pos: + c = src[pos] + if c == "\x00": + return None, None + if c == "\\": + if pos + 1 < max_pos: + title.append(src[pos : pos + 2]) + pos += 2 + continue + return None, None + if c == closer: + return unescape_char("".join(title)), pos + 1 + title.append(src[pos]) + pos += 1 return None, None @@ -182,12 +202,46 @@ def parse_link(src: str, pos: int) -> Un assert href_pos is not None title, title_pos = parse_link_title(src, href_pos, len(src)) next_pos = title_pos or href_pos - m = PAREN_END_RE.match(src, next_pos) - if not m: + next_pos = _skip_ascii_whitespace(src, next_pos) + if next_pos >= len(src) or src[next_pos] != ")": return None, None href = unescape_char(href) attrs = {"url": escape_url(href)} if title: attrs["title"] = title - return attrs, m.end() + return attrs, next_pos + 1 + + +def _skip_ascii_whitespace(src: str, pos: int, max_pos: Union[int, None] = None) -> int: + if max_pos is None: + max_pos = len(src) + while pos < max_pos and src[pos] in ASCII_WHITESPACE: + pos += 1 + return pos + + +def _skip_link_start_whitespace(src: str, pos: int) -> int: + while pos < len(src) and src[pos] in " \t": + pos += 1 + if pos < len(src) and src[pos] in "\n\r": + if src[pos] == "\r" and pos + 1 < len(src) and src[pos + 1] == "\n": + pos += 2 + else: + pos += 1 + while pos < len(src) and src[pos] in " \t": + pos += 1 + return pos + + +def _parse_angle_link_href(src: str, pos: int) -> Union[Tuple[str, int], Tuple[None, None]]: + start = pos + 1 + pos = start + while pos < len(src): + c = src[pos] + if c == ">": + return src[start:pos], pos + 1 + if c in "<\\\n\r\x00": + return None, None + pos += 1 + return None, None Index: mistune-3.1.3/src/mistune/inline_parser.py =================================================================== --- mistune-3.1.3.orig/src/mistune/inline_parser.py +++ mistune-3.1.3/src/mistune/inline_parser.py @@ -1,4 +1,5 @@ import re +from dataclasses import dataclass from typing import ( Any, Dict, @@ -6,22 +7,23 @@ from typing import ( Match, MutableMapping, Optional, + Set, + Tuple, ) from .core import InlineState, Parser from .helpers import ( HTML_ATTRIBUTES, HTML_TAGNAME, - PREVENT_BACKSLASH, PUNCTUATION, parse_link, parse_link_label, - parse_link_text, unescape_char, ) from .util import escape_url, unikey -PAREN_END_RE = re.compile(r"\s*\)") +_REGEX_META_CHARS = set(r"()[]{}?*+|.^$") +_CHARREF_PREFIX = re.compile(r"(#[0-9]{1,7};|#[xX][0-9a-fA-F]+;|[^\t\n\f <&#;]{1,32};)") AUTO_EMAIL = ( r"""<[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9]""" @@ -38,15 +40,6 @@ INLINE_HTML = ( r"" # cdata ) -EMPHASIS_END_RE = { - "*": re.compile(r"(?:" + PREVENT_BACKSLASH + r"\\\*|[^\s*])\*(?!\*)"), - "_": re.compile(r"(?:" + PREVENT_BACKSLASH + r"\\_|[^\s_])_(?!_)\b"), - "**": re.compile(r"(?:" + PREVENT_BACKSLASH + r"\\\*|[^\s*])\*\*(?!\*)"), - "__": re.compile(r"(?:" + PREVENT_BACKSLASH + r"\\_|[^\s_])__(?!_)\b"), - "***": re.compile(r"(?:" + PREVENT_BACKSLASH + r"\\\*|[^\s*])\*\*\*(?!\*)"), - "___": re.compile(r"(?:" + PREVENT_BACKSLASH + r"\\_|[^\s_])___(?!_)\b"), -} - class InlineParser(Parser[InlineState]): sc_flag = 0 @@ -92,6 +85,7 @@ class InlineParser(Parser[InlineState]): super(InlineParser, self).__init__() self.hard_wrap = hard_wrap + self._fast_trigger_chars: Optional[Set[str]] = None # lazy add linebreak if hard_wrap: self.specification["linebreak"] = self.HARD_LINEBREAK @@ -100,15 +94,20 @@ class InlineParser(Parser[InlineState]): self._methods = {name: getattr(self, "parse_" + name) for name in self.rules} + def register( + self, + name: str, + pattern: Optional[str], + func: Any, + before: Optional[str] = None, + ) -> None: + super().register(name, pattern, func, before=before) + self._fast_trigger_chars = None + def parse_escape(self, m: Match[str], state: InlineState) -> int: text = m.group(0) text = unescape_char(text) - state.append_token( - { - "type": "text", - "raw": text, - } - ) + self.process_text(text, state, parse_emphasis=False) return m.end() def parse_link(self, m: Match[str], state: InlineState) -> Optional[int]: @@ -116,10 +115,7 @@ class InlineParser(Parser[InlineState]): marker = m.group(0) is_image = marker[0] == "!" - if is_image and state.in_image: - state.append_token({"type": "text", "raw": marker}) - return pos - elif not is_image and state.in_link: + if not is_image and state.in_link: state.append_token({"type": "text", "raw": marker}) return pos @@ -129,7 +125,7 @@ class InlineParser(Parser[InlineState]): if pos <= state.no_close_bracket_before: state.append_token({"type": "text", "raw": marker}) return pos - text, end_pos = parse_link_text(state.src, pos) + text, end_pos = _parse_link_text(state, pos) if text is None: if end_pos and end_pos > state.no_close_bracket_before: state.no_close_bracket_before = end_pos @@ -145,6 +141,9 @@ class InlineParser(Parser[InlineState]): if end_pos >= len(state.src) and label is None: return None + if not is_image and self._contains_nested_link(text, state): + return None + rules = ["codespan", "prec_auto_link", "prec_inline_html"] prec_pos = self.precedence_scan(m, state, end_pos, rules) if prec_pos: @@ -186,6 +185,28 @@ class InlineParser(Parser[InlineState]): return end_pos return None + def _contains_nested_link(self, text: str, state: InlineState) -> bool: + if "[" not in text: + return False + + sc = self.compile_sc(["link"]) + pos = 0 + while pos < len(text): + m = sc.search(text, pos) + if not m: + return False + + marker = m.group(0) + if marker == "[" and (m.start() == 0 or text[m.start() - 1] != "!"): + nested_state = state.copy() + nested_state.src = text + new_pos = self.parse_link(m, nested_state) + if new_pos and any(token["type"] == "link" for token in nested_state.tokens): + return True + pos = m.start() + 1 + + return False + def __parse_link_token( self, is_image: bool, @@ -244,52 +265,8 @@ class InlineParser(Parser[InlineState]): ) def parse_emphasis(self, m: Match[str], state: InlineState) -> int: - pos = m.end() - - marker = m.group(0) - mlen = len(marker) - if mlen == 1 and state.in_emphasis: - state.append_token({"type": "text", "raw": marker}) - return pos - elif mlen == 2 and state.in_strong: - state.append_token({"type": "text", "raw": marker}) - return pos - - _end_re = EMPHASIS_END_RE[marker] - m1 = _end_re.search(state.src, pos) - if not m1: - state.append_token({"type": "text", "raw": marker}) - return pos - - end_pos = m1.end() - text = state.src[pos : end_pos - mlen] - - prec_pos = self.precedence_scan(m, state, end_pos) - if prec_pos: - return prec_pos - - new_state = state.copy() - new_state.src = text - if mlen == 1: - new_state.in_emphasis = True - children = self.render(new_state) - state.append_token({"type": "emphasis", "children": children}) - elif mlen == 2: - new_state.in_strong = True - children = self.render(new_state) - state.append_token({"type": "strong", "children": children}) - else: - new_state.in_emphasis = True - new_state.in_strong = True - - children = [{"type": "strong", "children": self.render(new_state)}] - state.append_token( - { - "type": "emphasis", - "children": children, - } - ) - return end_pos + self.process_text(m.group(0), state) + return m.end() def parse_codespan(self, m: Match[str], state: InlineState) -> int: marker = m.group(0) @@ -331,15 +308,41 @@ class InlineParser(Parser[InlineState]): state.in_link = False return end_pos - def process_text(self, text: str, state: InlineState) -> None: - state.append_token({"type": "text", "raw": text}) + def process_text(self, text: str, state: InlineState, parse_emphasis: bool = True) -> None: + if ( + parse_emphasis + and state.tokens + and state.tokens[-1]["type"] == "text" + and state.tokens[-1].get("_emphasis", True) + and not _is_entity_boundary(state.tokens[-1]["raw"], text) + ): + state.tokens[-1]["raw"] += text + else: + token: Dict[str, Any] = {"type": "text", "raw": text} + if not parse_emphasis: + token["_emphasis"] = False + state.append_token(token) def parse(self, state: InlineState) -> List[Dict[str, Any]]: pos = 0 sc = self.compile_sc() while pos < len(state.src): - m = sc.search(state.src, pos) + fast_end = self._find_fast_text_end(state.src, pos) + if fast_end is None: + m = sc.search(state.src, pos) + else: + if fast_end > pos: + self.process_text(state.src[pos:fast_end], state) + pos = fast_end + if pos >= len(state.src): + break + m = sc.match(state.src, pos) + if not m: + if fast_end is not None: + self.process_text(state.src[pos : pos + 1], state) + pos += 1 + continue break end_pos = m.start() @@ -361,8 +364,57 @@ class InlineParser(Parser[InlineState]): self.process_text(state.src, state) elif pos < len(state.src): self.process_text(state.src[pos:], state) + state.tokens = _finalize_emphasis_tokens(state.tokens, "emphasis" in self.rules) return state.tokens + def _find_fast_text_end(self, src: str, pos: int) -> Optional[int]: + chars = self._get_fast_trigger_chars() + if chars is None: + return None + + end_pos: Optional[int] = None + for c in chars: + if c == "\n": + continue + p = src.find(c, pos) + if p != -1 and (end_pos is None or p < end_pos): + end_pos = p + + if "\n" in chars: + p = src.find("\n", pos) + if p != -1: + p = self._find_linebreak_start(src, pos, p) + if end_pos is None or p < end_pos: + end_pos = p + + if end_pos is None: + return len(src) + return end_pos + + def _find_linebreak_start(self, src: str, min_pos: int, newline_pos: int) -> int: + pos = newline_pos + while pos > min_pos and src[pos - 1] == " ": + pos -= 1 + if pos == newline_pos and pos > min_pos and src[pos - 1] == "\\": + return pos - 1 + return pos + + def _get_fast_trigger_chars(self) -> Optional[Set[str]]: + chars = self._fast_trigger_chars + if chars is not None: + return chars + + chars = set() + for name in self.rules: + pattern = self.specification.get(name) + rule_chars = _get_rule_start_chars(name, pattern) + if rule_chars is None: + self._fast_trigger_chars = None + return None + chars.update(rule_chars) + self._fast_trigger_chars = chars + return chars + def precedence_scan( self, m: Match[str], @@ -409,3 +461,357 @@ class InlineParser(Parser[InlineState]): state = self.state_cls(env) state.src = s return self.render(state) + + +def _get_rule_start_chars(name: str, pattern: Optional[str]) -> Optional[Set[str]]: + known = { + "escape": {"\\"}, + "codespan": {"`"}, + "emphasis": {"*", "_"}, + "link": {"!", "["}, + "auto_link": {"<"}, + "auto_email": {"<"}, + "inline_html": {"<"}, + "linebreak": {"\n"}, + "softbreak": {"\n"}, + "prec_auto_link": {"<"}, + "prec_inline_html": {"<"}, + # built-in plugins + "url_link": {"h"}, + "strikethrough": {"~"}, + "mark": {"="}, + "insert": {"^"}, + "superscript": {"^"}, + "subscript": {"~"}, + "footnote": {"["}, + "inline_math": {"$"}, + "ruby": {"["}, + "inline_spoiler": {">"}, + } + if name in known: + return known[name] + if not pattern: + return set() + return _guess_pattern_start_chars(pattern) + + +def _guess_pattern_start_chars(pattern: str) -> Optional[Set[str]]: + if not pattern: + return set() + + if pattern.startswith("\\") and len(pattern) > 1: + c = pattern[1] + if c in _REGEX_META_CHARS or c in PUNCTUATION: + return {c} + return None + + c = pattern[0] + if c in _REGEX_META_CHARS or c.isspace(): + return None + return {c} + + +def _is_entity_boundary(left: str, right: str) -> bool: + return left.endswith("&") and _CHARREF_PREFIX.match(right) is not None + + +@dataclass +class _Delimiter: + index: int + marker: str + length: int + can_open: bool + can_close: bool + + +def _finalize_emphasis_tokens(tokens: List[Dict[str, Any]], enabled: bool) -> List[Dict[str, Any]]: + if not enabled: + return _clean_emphasis_tokens(tokens) + if not _contains_emphasis_marker(tokens): + return _clean_emphasis_tokens(tokens) + + parts: List[Dict[str, Any]] = [] + delimiters: List[_Delimiter] = [] + source = _emphasis_source_text(tokens) + source_pos = 0 + for token in tokens: + if token["type"] == "text" and token.get("_emphasis", True): + _split_text_token(token, source, source_pos, parts, delimiters) + else: + parts.append(_clean_emphasis_token(token)) + source_pos += _emphasis_source_length(token) + + _process_emphasis_delimiters(parts, delimiters) + return _merge_text_tokens(parts) + + +def _contains_emphasis_marker(tokens: List[Dict[str, Any]]) -> bool: + for token in tokens: + if token["type"] == "text" and token.get("_emphasis", True): + raw = token["raw"] + if "*" in raw or "_" in raw: + return True + return False + + +def _clean_emphasis_tokens(tokens: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + return [_clean_emphasis_token(token) for token in tokens] + + +def _clean_emphasis_token(token: Dict[str, Any]) -> Dict[str, Any]: + if "_emphasis" not in token: + return token + token = token.copy() + token.pop("_emphasis", None) + return token + + +def _emphasis_source_text(tokens: List[Dict[str, Any]]) -> str: + values = [] + for token in tokens: + if token["type"] == "text": + values.append(token["raw"]) + elif token["type"] in ("softbreak", "linebreak"): + values.append("\n") + else: + values.append("\ufffc") + return "".join(values) + + +def _emphasis_source_length(token: Dict[str, Any]) -> int: + if token["type"] == "text": + return len(token["raw"]) + return 1 + + +def _split_text_token( + token: Dict[str, Any], + source: str, + source_start: int, + parts: List[Dict[str, Any]], + delimiters: List[_Delimiter], +) -> None: + text = token["raw"] + pos = 0 + while pos < len(text): + if text[pos] not in "*_": + end = _next_delimiter_run(text, pos) + parts.append({"type": "text", "raw": text[pos:end]}) + pos = end + continue + + marker = text[pos] + end = pos + while end < len(text) and text[end] == marker: + end += 1 + length = end - pos + absolute = source_start + pos + can_open = _can_open_emphasis(source, absolute, length, marker) + can_close = _can_close_emphasis(source, absolute, length, marker) + index = len(parts) + parts.append({"type": "text", "raw": text[pos:end]}) + if can_open or can_close: + delimiters.append(_Delimiter(index, marker, length, can_open, can_close)) + pos = end + + +def _next_delimiter_run(text: str, pos: int) -> int: + while pos < len(text) and text[pos] not in "*_": + pos += 1 + return pos + + +def _process_emphasis_delimiters(parts: List[Dict[str, Any]], delimiters: List[_Delimiter]) -> None: + closer_pos = 0 + openers_bottom: Dict[Tuple[str, int, bool], int] = {} + while closer_pos < len(delimiters): + closer = delimiters[closer_pos] + if not closer.can_close or closer.length == 0: + closer_pos += 1 + continue + + opener_key = (closer.marker, closer.length % 3, closer.can_open) + opener_pos = closer_pos - 1 + opener_bottom = openers_bottom.get(opener_key, 0) + opener = None + while opener_pos >= opener_bottom: + candidate = delimiters[opener_pos] + if ( + candidate.marker == closer.marker + and candidate.can_open + and candidate.length > 0 + and _can_match_emphasis_delimiters(candidate, closer) + ): + opener = candidate + break + opener_pos -= 1 + + if opener is None: + openers_bottom[opener_key] = closer_pos + closer_pos += 1 + continue + + if opener.length >= 2 and closer.length >= 2: + use_length = 2 + else: + use_length = 1 + if use_length == 2 and not _has_strong_enabled(parts, opener, closer): + use_length = 1 + if use_length == 1 and not _has_emphasis_enabled(parts, opener, closer): + closer_pos += 1 + continue + if not _has_emphasis_content(parts, opener.index + 1, closer.index): + closer_pos += 1 + continue + + opener_text = parts[opener.index] + closer_text = parts[closer.index] + if opener_text["type"] != "text" or closer_text["type"] != "text": + closer_pos += 1 + continue + + opener_text["raw"] = opener_text["raw"][:-use_length] + closer_text["raw"] = closer_text["raw"][use_length:] + children = parts[opener.index + 1 : closer.index] + if use_length == 2: + node = {"type": "strong", "children": children} + else: + node = {"type": "emphasis", "children": children} + + old_closer_index = closer.index + parts[opener.index + 1 : old_closer_index] = [node] + + removed = old_closer_index - opener.index - 2 + closer.index = opener.index + 2 + if removed: + for delimiter in delimiters: + if opener.index < delimiter.index < old_closer_index: + delimiter.length = 0 + elif delimiter.index >= old_closer_index: + delimiter.index -= removed + + opener.length -= use_length + closer.length -= use_length + if opener.length == 0: + opener.can_open = False + if closer.length == 0: + closer.can_close = False + + if opener.can_open or closer.can_close: + closer_pos = max(opener_pos, openers_bottom.get(opener_key, 0)) + else: + closer_pos += 1 + + +def _has_strong_enabled(parts: List[Dict[str, Any]], opener: _Delimiter, closer: _Delimiter) -> bool: + return len(_text_raw(parts[opener.index])) >= 2 and len(_text_raw(parts[closer.index])) >= 2 + + +def _has_emphasis_enabled(parts: List[Dict[str, Any]], opener: _Delimiter, closer: _Delimiter) -> bool: + return bool(_text_raw(parts[opener.index]) and _text_raw(parts[closer.index])) + + +def _text_raw(token: Dict[str, Any]) -> str: + if token["type"] == "text": + return token["raw"] + return "" + + +def _has_emphasis_content(parts: List[Dict[str, Any]], start: int, end: int) -> bool: + for part in parts[start:end]: + if part["type"] != "text" or part["raw"] != "": + return True + return False + + +def _can_match_emphasis_delimiters(opener: _Delimiter, closer: _Delimiter) -> bool: + if opener.can_close or closer.can_open: + return (opener.length + closer.length) % 3 != 0 or opener.length % 3 == 0 and closer.length % 3 == 0 + return True + + +def _can_open_emphasis(text: str, start: int, size: int, marker: str) -> bool: + if start > 0: + previous = text[start - 1] + else: + previous = "\n" + if start + size < len(text): + next_char = text[start + size] + else: + next_char = "\n" + if marker == "_" and previous.isalnum() and next_char.isalnum(): + return False + if next_char.isspace(): + return False + if _is_punctuation(next_char) and not previous.isspace() and not _is_punctuation(previous): + return False + return True + + +def _can_close_emphasis(text: str, start: int, size: int, marker: str) -> bool: + if start > 0: + previous = text[start - 1] + else: + previous = "\n" + if start + size < len(text): + next_char = text[start + size] + else: + next_char = "\n" + if marker == "_" and previous.isalnum() and next_char.isalnum(): + return False + if previous.isspace(): + return False + if _is_punctuation(previous) and not next_char.isspace() and not _is_punctuation(next_char): + return False + return True + + +def _is_punctuation(c: str) -> bool: + return not c.isspace() and not c.isalnum() + + +def _merge_text_tokens(tokens: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + result: List[Dict[str, Any]] = [] + for token in tokens: + if token["type"] == "text" and token["raw"] == "": + continue + if token["type"] == "text" and result and result[-1]["type"] == "text": + if not _is_entity_boundary(result[-1]["raw"], token["raw"]): + result[-1]["raw"] += token["raw"] + continue + result.append(_clean_emphasis_token(token)) + return result + + +def _parse_link_text(state: InlineState, pos: int) -> Tuple[Optional[str], int]: + close_pos = _find_closing_bracket(state, pos) + if close_pos is None: + return None, len(state.src) + return state.src[pos:close_pos], close_pos + 1 + + +def _find_closing_bracket(state: InlineState, pos: int) -> Optional[int]: + cache = state.link_brackets.get(id(state.src)) + if cache is not None and cache[0] is state.src: + return cache[1].get(pos) + + pairs = _build_closing_bracket_map(state.src) + state.link_brackets[id(state.src)] = (state.src, pairs) + return pairs.get(pos) + + +def _build_closing_bracket_map(src: str) -> Dict[int, int]: + pairs: Dict[int, int] = {} + stack: List[int] = [] + pos = 0 + while pos < len(src): + c = src[pos] + if c == "\\": + pos += 2 + continue + if c == "[": + stack.append(pos + 1) + elif c == "]" and stack: + pairs[stack.pop()] = pos + pos += 1 + return pairs Index: mistune-3.1.3/src/mistune/list_parser.py =================================================================== --- mistune-3.1.3.orig/src/mistune/list_parser.py +++ mistune-3.1.3/src/mistune/list_parser.py @@ -1,8 +1,9 @@ """because list is complex, split list parser in a new file""" import re -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, Match -from .util import expand_leading_tab, expand_tab, strip_end +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Iterable, Optional, Match, Pattern, Dict, List, Tuple +from .util import strip_end if TYPE_CHECKING: from .block_parser import BlockParser @@ -17,9 +18,38 @@ LIST_PATTERN = ( _LINE_HAS_TEXT = re.compile(r"(\s*)\S") +@dataclass +class _ListMarker: + spaces: str + marker: str + text: str + + @property + def leading_width(self) -> int: + return len(self.spaces) + len(self.marker) + + @property + def bullet(self) -> str: + return self.marker[-1] + + @property + def ordered(self) -> bool: + return len(self.marker) > 1 + + +@dataclass +class _ListItemLines: + src: str + next_item: Optional[_ListMarker] = None + loose: bool = False + end_pos: Optional[int] = None + token_index: Optional[int] = None + + def parse_list(block: "BlockParser", m: Match[str], state: "BlockState") -> int: """Parse tokens for ordered and unordered list.""" - text = m.group("list_3") + item = _create_list_marker(m, "list") + text = item.text if not text.strip(): # Example 285 # an empty list item cannot interrupt a paragraph @@ -27,20 +57,19 @@ def parse_list(block: "BlockParser", m: if end_pos: return end_pos - marker = m.group("list_2") - ordered = len(marker) > 1 + marker = item.marker depth = state.depth() token: Dict[str, Any] = { "type": "list", "children": [], "tight": True, - "bullet": marker[-1], + "bullet": item.bullet, "attrs": { "depth": depth, - "ordered": ordered, + "ordered": item.ordered, }, } - if ordered: + if item.ordered: start = int(marker[:-1]) if start != 1: # Example 304 @@ -51,7 +80,7 @@ def parse_list(block: "BlockParser", m: token["attrs"]["start"] = start state.cursor = m.end() + 1 - groups: Optional[Tuple[str, str, str]] = (m.group("list_1"), marker, text) + item_or_none: Optional[_ListMarker] = item if depth >= block.max_nested_level - 1: rules = list(block.list_rules) @@ -59,9 +88,9 @@ def parse_list(block: "BlockParser", m: else: rules = block.list_rules - bullet = _get_list_bullet(marker[-1]) - while groups: - groups = _parse_list_item(block, bullet, groups, token, state, rules) + bullet = _get_list_bullet(item.bullet) + while item_or_none: + item_or_none = _parse_list_item(block, bullet, item_or_none, token, state, rules) end_pos = token.pop("_end_pos", None) _transform_tight_list(token) @@ -88,68 +117,83 @@ def _transform_tight_list(token: Dict[st def _parse_list_item( block: "BlockParser", bullet: str, - groups: Tuple[str, str, str], + item: _ListMarker, token: Dict[str, Any], state: "BlockState", rules: List[str], -) -> Optional[Tuple[str, str, str]]: - spaces, marker, text = groups - - leading_width = len(spaces) + len(marker) +) -> _ListMarker | None: + text = item.text + leading_width = item.leading_width text, continue_width = _compile_continue_width(text, leading_width) - item_pattern = _compile_list_item_pattern(bullet, leading_width) - pairs = [ - ("thematic_break", block.specification["thematic_break"]), - ("fenced_code", block.specification["fenced_code"]), - ("atx_heading", block.specification["atx_heading"]), - ("block_quote", block.specification["block_quote"]), - ("block_html", block.specification["block_html"]), - ("list", block.specification["list"]), - ] - if leading_width < 3: - _repl_w = str(leading_width) - pairs = [(n, p.replace("3", _repl_w, 1)) for n, p in pairs] + list_item_re = re.compile(_compile_list_item_pattern(bullet, leading_width)) + break_sc = _compile_list_break_sc(block, leading_width) - pairs.insert(1, ("list_item", item_pattern)) - regex = "|".join(r"(?P<%s>(?<=\n)%s)" % pair for pair in pairs) - sc = re.compile(regex, re.M) + lines = _collect_list_item_lines(block, list_item_re, break_sc, state, text, continue_width) + if lines.loose: + token["tight"] = False + if lines.end_pos is not None: + token["_tok_index"] = lines.token_index + token["_end_pos"] = lines.end_pos + child = state.child_state(_build_list_item_source(text, lines.src, continue_width)) + + block.parse(child, rules) + + if token["tight"] and _is_loose_list(child.tokens): + token["tight"] = False + + token["children"].append( + { + "type": "list_item", + "children": child.tokens, + } + ) + if lines.next_item: + return lines.next_item + + return None + + +def _collect_list_item_lines( + block: "BlockParser", + list_item_re: Pattern[str], + break_sc: Pattern[str], + state: "BlockState", + text: str, + continue_width: int, +) -> _ListItemLines: src = "" - next_group = None + next_item = None prev_blank_line = False - pos = state.cursor - - continue_space = " " * continue_width - while pos < state.cursor_max: - pos = state.find_line_end() - line = state.get_text(pos) - if block.BLANK_LINE.match(line): + while state.cursor < state.cursor_max: + raw_line = state.get_line(state.cursor) + next_pos = state.cursor + len(raw_line) + if block.BLANK_LINE.match(raw_line): src += "\n" prev_blank_line = True - state.cursor = pos + state.cursor = next_pos continue - line = expand_leading_tab(line) - if line.startswith(continue_space): + has_continuation = _has_continuation_indent(raw_line, continue_width) + if has_continuation: if prev_blank_line and not text and not src.strip(): # Example 280 # A list item can begin with at most one blank line break - src += line + src += raw_line prev_blank_line = False - state.cursor = pos + state.cursor = next_pos continue - m = sc.match(state.src, state.cursor) - if m: - tok_type = m.lastgroup + line = _expand_leading_tabs(raw_line) + line_break = _match_list_item_break(list_item_re, break_sc, state, line) + if line_break: + tok_type, m = line_break if tok_type == "list_item": - if prev_blank_line: - token["tight"] = False - next_group = (m.group("listitem_1"), m.group("listitem_2"), m.group("listitem_3")) - state.cursor = m.end() + 1 - break + next_item = _create_list_marker(m, "listitem") + state.cursor = next_pos + return _ListItemLines(src, next_item=next_item, loose=prev_blank_line) if tok_type == "list": break @@ -157,34 +201,73 @@ def _parse_list_item( tok_index = len(state.tokens) end_pos = block.parse_method(m, state) if end_pos: - token["_tok_index"] = tok_index - token["_end_pos"] = end_pos - break + return _ListItemLines(src, end_pos=end_pos, token_index=tok_index) - if prev_blank_line and not line.startswith(continue_space): + if prev_blank_line and not has_continuation: # not a continue line, and previous line is blank break - src += line - state.cursor = pos + src += raw_line + state.cursor = next_pos + + return _ListItemLines(src) + + +def _create_list_marker(m: Match[str], prefix: str) -> _ListMarker: + return _ListMarker( + spaces=m.group(prefix + "_1"), + marker=m.group(prefix + "_2"), + text=m.group(prefix + "_3"), + ) + +def _build_list_item_source(text: str, src: str, continue_width: int) -> str: text += _clean_list_item_text(src, continue_width) - child = state.child_state(strip_end(text)) + return strip_end(text) - block.parse(child, rules) - if token["tight"] and _is_loose_list(child.tokens): - token["tight"] = False +def _compile_list_break_sc(block: "BlockParser", leading_width: int) -> Pattern[str]: + pairs = [(name, block.specification[name]) for name in _get_list_break_rules(block)] + if leading_width < 3: + _repl_w = str(leading_width) + pairs = [(n, p.replace("3", _repl_w, 1)) for n, p in pairs] - token["children"].append( - { - "type": "list_item", - "children": child.tokens, - } - ) - if next_group: - return next_group + regex = "|".join(r"(?P<%s>(?<=\n)%s)" % pair for pair in pairs) + return re.compile(regex, re.M) + +def _get_list_break_rules(block: "BlockParser") -> list[str]: + rules = [ + "thematic_break", + "fenced_code", + "atx_heading", + "block_quote", + "block_html", + "list", + ] + if "fenced_directive" in block.specification: + rules.insert(1, "fenced_directive") + return rules + + +def _match_list_item_break( + list_item_re: Pattern[str], + break_sc: Pattern[str], + state: "BlockState", + line: str, +) -> tuple[str, Match[str]] | None: + m = break_sc.match(state.src, state.cursor) + if m and m.lastgroup == "thematic_break": + return "thematic_break", m + + m2 = list_item_re.match(line) + if m2: + return "list_item", m2 + + if m: + tok_type = m.lastgroup + assert tok_type is not None + return tok_type, m return None @@ -213,16 +296,16 @@ def _compile_list_item_pattern(bullet: s def _compile_continue_width(text: str, leading_width: int) -> Tuple[str, int]: - text = expand_leading_tab(text, 3) - text = expand_tab(text) + text = _expand_leading_tabs(text, leading_width) m2 = _LINE_HAS_TEXT.match(text) if m2: # indent code, startswith 5 spaces - if text.startswith(" "): + indent = _count_indent(text) + if indent >= 5: space_width = 1 else: - space_width = len(m2.group(1)) + space_width = indent text = text[space_width:] + "\n" else: @@ -234,23 +317,59 @@ def _compile_continue_width(text: str, l def _clean_list_item_text(src: str, continue_width: int) -> str: - # according to Example 7, tab should be treated as 3 spaces rv = [] - trim_space = " " * continue_width lines = src.split("\n") for line in lines: - if line.startswith(trim_space): - line = line.replace(trim_space, "", 1) - # according to CommonMark Example 5 - # tab should be treated as 4 spaces - line = expand_tab(line) - rv.append(line) + if _has_continuation_indent(line, continue_width): + rv.append(_strip_continuation_indent(line, continue_width)) else: - rv.append(line) + rv.append(_expand_leading_tabs(line)) return "\n".join(rv) +def _has_continuation_indent(line: str, columns: int) -> bool: + return _count_indent(line) >= columns + + +def _strip_continuation_indent(line: str, columns: int) -> str: + expanded = _expand_leading_tabs(line) + if len(expanded) >= columns: + return expanded[columns:] + return "" + + +def _expand_leading_tabs(line: str, start_column: int = 0) -> str: + column = start_column + parts = [] + index = 0 + while index < len(line): + c = line[index] + if c == " ": + parts.append(" ") + column += 1 + elif c == "\t": + size = 4 - column % 4 + parts.append(" " * size) + column += size + else: + break + index += 1 + return "".join(parts) + line[index:] + + +def _count_indent(text: str) -> int: + column = 0 + for c in text: + if c == " ": + column += 1 + elif c == "\t": + column += 4 - column % 4 + else: + break + return column + + def _is_loose_list(tokens: Iterable[Dict[str, Any]]) -> bool: paragraph_count = 0 for tok in tokens: Index: mistune-3.1.3/src/mistune/plugins/def_list.py =================================================================== --- mistune-3.1.3.orig/src/mistune/plugins/def_list.py +++ mistune-3.1.3/src/mistune/plugins/def_list.py @@ -1,5 +1,5 @@ import re -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Match +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Match, Optional, Tuple from ..util import strip_end @@ -12,76 +12,43 @@ __all__ = ["def_list"] # https://michelf.ca/projects/php-markdown/extra/#def-list -DEF_PATTERN = ( - r"^(?P(?:[^\n]+\n)+?)" - r"\n?(?:" - r"\:[ \t]+.*\n" - r"(?:[^\n]+\n)*" # lazy continue line - r"(?:(?:[ \t]*\n)*[ \t]+[^\n]+\n)*" - r"(?:[ \t]*\n)*" - r")+" -) -DEF_RE = re.compile(DEF_PATTERN, re.M) +DEF_PATTERN = r"^:[ \t]+.*(?:\n|$)" DD_START_RE = re.compile(r"^:[ \t]+", re.M) TRIM_RE = re.compile(r"^ {0,4}", re.M) HAS_BLANK_LINE_RE = re.compile(r"\n[ \t]*\n$") -def parse_def_list(block: "BlockParser", m: Match[str], state: "BlockState") -> int: - pos = m.end() - children = list(_parse_def_item(block, m)) - - m2 = DEF_RE.match(state.src, pos) - while m2: - children.extend(list(_parse_def_item(block, m2))) - pos = m2.end() - m2 = DEF_RE.match(state.src, pos) - - state.append_token( - { - "type": "def_list", - "children": children, - } - ) - return pos - - -def _parse_def_item(block: "BlockParser", m: Match[str]) -> Iterable[Dict[str, Any]]: - head = m.group("def_list_head") +def parse_def_list(block: "BlockParser", m: Match[str], state: "BlockState") -> Optional[int]: + head = _get_previous_paragraph(state) + if head is None: + return None + + definitions, end_pos = _collect_definitions(state.src, state.cursor, state.cursor_max, head[1]) + if not definitions: + return None + + children = list(_parse_def_item(block, head[0], definitions)) + _replace_previous_paragraph(state, children) + return end_pos + + +def _parse_def_item( + block: "BlockParser", + head: str, + definitions: List[Tuple[str, bool]], +) -> Iterable[Dict[str, Any]]: for line in head.splitlines(): yield { "type": "def_list_head", "text": line, } - src = m.group(0) - end = len(head) - - m2 = DD_START_RE.search(src, end) - assert m2 is not None - start = m2.start() - prev_blank_line = src[end:start] == "\n" - while m2: - m2 = DD_START_RE.search(src, start + 1) - if not m2: - break - - end = m2.start() - text = src[start:end].replace(":", " ", 1) - children = _process_text(block, text, prev_blank_line) - prev_blank_line = bool(HAS_BLANK_LINE_RE.search(text)) + for text, loose in definitions: + children = _process_text(block, text, loose) yield { "type": "def_list_item", "children": children, } - start = end - - text = src[start:].replace(":", " ", 1) - children = _process_text(block, text, prev_blank_line) - yield { - "type": "def_list_item", - "children": children, - } def _process_text(block: "BlockParser", text: str, loose: bool) -> List[Any]: @@ -96,6 +63,91 @@ def _process_text(block: "BlockParser", return tokens +def _get_previous_paragraph(state: "BlockState") -> Optional[Tuple[str, bool]]: + if not state.tokens: + return None + + last_token = state.tokens[-1] + if last_token["type"] == "paragraph": + return last_token["text"], False + + if last_token["type"] == "blank_line" and len(state.tokens) > 1: + prev_token = state.tokens[-2] + if prev_token["type"] == "paragraph": + return prev_token["text"], True + + return None + + +def _replace_previous_paragraph(state: "BlockState", children: List[Dict[str, Any]]) -> None: + if state.tokens[-1]["type"] == "blank_line": + state.tokens.pop() + state.tokens.pop() + + if state.tokens and state.tokens[-1]["type"] == "def_list": + state.tokens[-1]["children"].extend(children) + else: + state.append_token( + { + "type": "def_list", + "children": children, + } + ) + + +def _collect_definitions(src: str, pos: int, max_pos: int, loose: bool) -> Tuple[List[Tuple[str, bool]], int]: + definitions = [] + while pos < max_pos: + line = _get_line(src, pos, max_pos) + if not DD_START_RE.match(line): + break + + start = pos + pos += len(line) + pos = _scan_definition_tail(src, pos, max_pos) + text = src[start:pos].replace(":", " ", 1) + definitions.append((text, loose)) + loose = bool(HAS_BLANK_LINE_RE.search(text)) + + return definitions, pos + + +def _scan_definition_tail(src: str, pos: int, max_pos: int) -> int: + while pos < max_pos: + line = _get_line(src, pos, max_pos) + if DD_START_RE.match(line): + break + + if line.strip(): + pos += len(line) + continue + + while pos < max_pos: + line = _get_line(src, pos, max_pos) + if line.strip(): + break + pos += len(line) + + if pos >= max_pos: + break + + line = _get_line(src, pos, max_pos) + if DD_START_RE.match(line): + break + if line.startswith((" ", "\t")): + continue + return pos + + return pos + + +def _get_line(src: str, pos: int, max_pos: int) -> str: + end = src.find("\n", pos, max_pos) + if end == -1: + return src[pos:max_pos] + return src[pos : end + 1] + + def render_def_list(renderer: "BaseRenderer", text: str) -> str: return "
\n" + text + "
\n" Index: mistune-3.1.3/src/mistune/plugins/speedup.py =================================================================== --- mistune-3.1.3.orig/src/mistune/plugins/speedup.py +++ mistune-3.1.3/src/mistune/plugins/speedup.py @@ -1,50 +1,16 @@ -import re -import string -from typing import TYPE_CHECKING, Match +from typing import TYPE_CHECKING if TYPE_CHECKING: - from ..block_parser import BlockParser - from ..core import BlockState, InlineState - from ..inline_parser import InlineParser from ..markdown import Markdown -# because mismatch is too slow, add parsers for paragraph and text - -HARD_LINEBREAK_RE = re.compile(r" *\n\s*") -PARAGRAPH = ( - # start with none punctuation, not number, not whitespace - r"(?:^[^\s\d" + re.escape(string.punctuation) + r"][^\n]*\n)+" -) - __all__ = ["speedup"] -def parse_text(inline: "InlineParser", m: Match[str], state: "InlineState") -> int: - text = m.group(0) - text = HARD_LINEBREAK_RE.sub("\n", text) - inline.process_text(text, state) - return m.end() - - -def parse_paragraph(block: "BlockParser", m: Match[str], state: "BlockState") -> int: - text = m.group(0) - state.add_paragraph(text) - return m.end() - - def speedup(md: "Markdown") -> None: - """Increase the speed of parsing paragraph and inline text.""" - md.block.register("paragraph", PARAGRAPH, parse_paragraph) - - punc = r"\\>!\s*(?P int: - text, end_pos = block.extract_block_quote(m, state) + text, end_pos, lazy_line_starts = block.extract_block_quote(m, state) if not text.endswith("\n"): # ensure it endswith \n to make sure # _BLOCK_SPOILER_MATCH.match works @@ -30,7 +30,7 @@ def parse_block_spoiler(block: "BlockPar tok_type = "block_quote" # scan children state - child = state.child_state(text) + child = state.child_state(text, lazy_line_starts=lazy_line_starts) if state.depth() >= block.max_nested_level - 1: rules = list(block.block_quote_rules) rules.remove("block_quote") Index: mistune-3.1.3/src/mistune/plugins/table.py =================================================================== --- mistune-3.1.3.orig/src/mistune/plugins/table.py +++ mistune-3.1.3/src/mistune/plugins/table.py @@ -10,8 +10,6 @@ from typing import ( Union, ) -from ..helpers import PREVENT_BACKSLASH - if TYPE_CHECKING: from ..block_parser import BlockParser from ..core import BaseRenderer, BlockState @@ -23,18 +21,12 @@ __all__ = ["table", "table_in_quote", "t TABLE_PATTERN = ( - r"^ {0,3}\|(?P.+)\|[ \t]*\n" - r" {0,3}\|(?P *[-:]+[-| :]*)\|[ \t]*\n" - r"(?P(?: {0,3}\|.*\|[ \t]*(?:\n|$))*)\n*" + r"^ {0,3}\|[^\n]*\|[ \t]*(?:\n|$)" ) NP_TABLE_PATTERN = ( - r"^ {0,3}(?P\S.*\|.*)\n" - r" {0,3}(?P[-:]+ *\|[-| :]*)\n" - r"(?P(?:.*\|.*(?:\n|$))*)\n*" + r"^ {0,3}\S[^\n]*\|[^\n]*(?:\n|$)" ) -TABLE_CELL = re.compile(r"^ {0,3}\|(.+)\|[ \t]*$") -CELL_SPLIT = re.compile(r" *" + PREVENT_BACKSLASH + r"\| *") ALIGN_CENTER = re.compile(r"^ *:-+: *$") ALIGN_LEFT = re.compile(r"^ *:-+ *$") ALIGN_RIGHT = re.compile(r"^ *-+: *$") @@ -42,23 +34,33 @@ ALIGN_RIGHT = re.compile(r"^ *-+: *$") def parse_table(block: "BlockParser", m: Match[str], state: "BlockState") -> Optional[int]: pos = m.end() - header = m.group("table_head") - align = m.group("table_align") + header = _strip_pipe_table_row(m.group(0)) + if header is None: + return None + + align_line = state.get_line(pos) + align = _strip_pipe_table_row(align_line) + if align is None: + return None + thead, aligns = _process_thead(header, align) if not thead: - return None + return _parse_invalid_pipe_table(state, pos + len(align_line)) assert aligns is not None + pos += len(align_line) rows = [] - body = m.group("table_body") - for text in body.splitlines(): - m2 = TABLE_CELL.match(text) - if not m2: # pragma: no cover - return None - row = _process_row(m2.group(1), aligns) + while pos < state.cursor_max: + line = state.get_line(pos) + text = _strip_pipe_table_row(line) + if text is None: + break + + row = _process_row(text, aligns) if not row: - return None + return _parse_invalid_pipe_table(state, pos + len(line)) rows.append(row) + pos += len(line) children = [thead, {"type": "table_body", "children": rows}] state.append_token({"type": "table", "children": children}) @@ -66,29 +68,43 @@ def parse_table(block: "BlockParser", m: def parse_nptable(block: "BlockParser", m: Match[str], state: "BlockState") -> Optional[int]: - header = m.group("nptable_head") - align = m.group("nptable_align") + pos = m.end() + header = _strip_table_line(m.group(0)) + if header is None: + return None + + align_line = state.get_line(pos) + align = _strip_table_line(align_line) + if align is None: + return None + thead, aligns = _process_thead(header, align) if not thead: return None assert aligns is not None + pos += len(align_line) rows = [] - body = m.group("nptable_body") - for text in body.splitlines(): + while pos < state.cursor_max: + line = state.get_line(pos) + text = _strip_table_line(line) + if text is None: + break + row = _process_row(text, aligns) if not row: return None rows.append(row) + pos += len(line) children = [thead, {"type": "table_body", "children": rows}] state.append_token({"type": "table", "children": children}) - return m.end() + return pos def _process_thead(header: str, align: str) -> Union[Tuple[None, None], Tuple[Dict[str, Any], List[str]]]: - headers = CELL_SPLIT.split(header) - aligns = CELL_SPLIT.split(align) + headers = _split_table_cells(header) + aligns = _split_table_cells(align) if len(headers) != len(aligns): return None, None @@ -111,7 +127,7 @@ def _process_thead(header: str, align: s def _process_row(text: str, aligns: List[str]) -> Optional[Dict[str, Any]]: - cells = CELL_SPLIT.split(text) + cells = _split_table_cells(text) if len(cells) != len(aligns): return None @@ -122,6 +138,54 @@ def _process_row(text: str, aligns: List return {"type": "table_row", "children": children} +def _strip_pipe_table_row(line: str) -> Optional[str]: + text = line.rstrip("\n").rstrip(" \t") + if not text.startswith("|") and text.startswith((" ", "\t")): + text = text.lstrip(" ") + if not text.startswith("|") or not text.endswith("|"): + return None + return text[1:-1] + + +def _parse_invalid_pipe_table(state: "BlockState", pos: int) -> int: + while pos < state.cursor_max: + line = state.get_line(pos) + if _strip_pipe_table_row(line) is None: + break + pos += len(line) + state.add_paragraph(state.src[state.cursor:pos]) + return pos + + +def _strip_table_line(line: str) -> Optional[str]: + text = line.rstrip("\n").rstrip(" \t") + if not text or "|" not in text: + return None + return text + + +def _split_table_cells(text: str) -> List[str]: + cells = [] + start = 0 + pos = 0 + while pos < len(text): + if text[pos] == "|" and not _is_escaped_pipe(text, pos): + cells.append(text[start:pos].strip()) + start = pos + 1 + pos += 1 + cells.append(text[start:].strip()) + return cells + + +def _is_escaped_pipe(text: str, pos: int) -> bool: + backslashes = 0 + pos -= 1 + while pos >= 0 and text[pos] == "\\": + backslashes += 1 + pos -= 1 + return backslashes % 2 == 1 + + def render_table(renderer: "BaseRenderer", text: str) -> str: return "\n" + text + "
\n" Index: mistune-3.1.3/src/mistune/util.py =================================================================== --- mistune-3.1.3.orig/src/mistune/util.py +++ mistune-3.1.3/src/mistune/util.py @@ -69,9 +69,11 @@ def unescape(s: str) -> str: _striptags_re = re.compile(r"(|<[^>]*>)") +_strip_image_re = re.compile(r"]*\balt=(\"([^\"]*)\"|'([^']*)')[^>]*>") def striptags(s: str) -> str: + s = _strip_image_re.sub(lambda m: m.group(2) or m.group(3) or "", s) return _striptags_re.sub("", s) Index: mistune-3.1.3/tests/fixtures/diff-commonmark.txt =================================================================== --- mistune-3.1.3.orig/tests/fixtures/diff-commonmark.txt +++ /dev/null @@ -1,76 +0,0 @@ -# Differences - -Results different than commonmark. - - -## Setext headings - - -Example: 93 - - -```````````````````````````````` example -> foo -bar -=== -. -
-

foo -bar

-
-```````````````````````````````` - -If the dash is less than 3, it is not a `
`, in this case, it -can be rendered into `

`. - -```````````````````````````````` example -- # Foo -- Bar - -- - baz -. -
    -
  • -

    Foo

    -
    • -
  • -

    Bar

    -baz
    • -
-```````````````````````````````` - -## Image - -Example 573 - -```````````````````````````````` example -![foo ![bar](/url)](/url2) -. -

foo ![bar](/url)

-```````````````````````````````` - -## Link - -Example 517 - -```````````````````````````````` example -[foo [bar](/uri)](/uri) -. -

foo [bar](/uri)

-```````````````````````````````` - -Example 518 - -```````````````````````````````` example -[foo *[bar [baz](/uri)](/uri)*](/uri) -. -

foo [bar [baz](/uri)](/uri)

-```````````````````````````````` - -Example 519 - -```````````````````````````````` example -![[[foo](uri1)](uri2)](uri3) -. -

[foo](uri1)

-```````````````````````````````` Index: mistune-3.1.3/tests/fixtures/fenced_admonition.txt =================================================================== --- mistune-3.1.3.orig/tests/fixtures/fenced_admonition.txt +++ mistune-3.1.3/tests/fixtures/fenced_admonition.txt @@ -115,11 +115,15 @@ Test with nested admonition

Hint

Danger

-
````{important}
-```{error}
-```
-````
-
+
+

Attention

+
+

Important

+
+

Error

+
+
+
Index: mistune-3.1.3/tests/fixtures/fix-commonmark.txt =================================================================== --- mistune-3.1.3.orig/tests/fixtures/fix-commonmark.txt +++ mistune-3.1.3/tests/fixtures/fix-commonmark.txt @@ -1,9 +1,9 @@ -Fix problems that commonmark has. +Historical CommonMark edge cases. ## Links -Links can't contain links. +Autolinks in link text stay literal. ```````````````````````````````` example [](/foo) @@ -19,53 +19,31 @@ Links can't contain links. ## Emphasis -`` doesn't contain ``, `` doesn't contain ``. +CommonMark delimiter handling is the default. ```````````````````````````````` example *_em_* __**strong**__ ______m______ . -

_em_ **strong** ______m______

+

em strong m

```````````````````````````````` -### Non aggressive emphasis - ```````````````````````````````` example *a **b c* d** . -

a **b c d**

-```````````````````````````````` - -While CommonMark would render it into: - -```

a b c d

-``` +```````````````````````````````` ```````````````````````````````` example *a **b c* d** . -

a **b c d**

-```````````````````````````````` - -While CommonMark would render it into: - -```

a b c d

-``` - -What if the string is: +```````````````````````````````` ```````````````````````````````` example *a **b c* d . -

a **b c d

-```````````````````````````````` - -CommonMark would still be a mess: - -```

*a *b c d

-``` +```````````````````````````````` ## Max depth @@ -80,7 +58,11 @@ CommonMark would still be a mess:
-

> > b

+
+
+

b

+
+
@@ -106,9 +88,13 @@ CommonMark would still be a mess:
  • c
    • d
      • e
          -
        • f -- g -- h
          • +
        • f
            +
          • g
              +
            • h
              • +
            +
            • +
          +
      Index: mistune-3.1.3/tests/fixtures/renderer_markdown.txt =================================================================== --- mistune-3.1.3.orig/tests/fixtures/renderer_markdown.txt +++ mistune-3.1.3/tests/fixtures/renderer_markdown.txt @@ -26,7 +26,7 @@ this is *em*, **strong**, and `code` [link]: /url "title" . -[link][link], [link]( "title") +[link][link], [link][link](https://foo(bar "title") [link]: /url "title" ```````````````````````````````` Index: mistune-3.1.3/tests/test_commonmark.py =================================================================== --- mistune-3.1.3.orig/tests/test_commonmark.py +++ mistune-3.1.3/tests/test_commonmark.py @@ -2,66 +2,10 @@ import mistune from tests import BaseTestCase, normalize_html -DIFF_CASES = { - "setext_headings_093", - "html_blocks_191", # mistune keeps \n - "images_573", # image can not be in image - "links_495", - "links_517", # aggressive link group - "links_518", - "links_519", - "links_531", - "links_532", -} - -IGNORE_CASES = { - # we don't support link title in (title) - "links_496", - "links_504", - "link_reference_definitions_202", - # we don't support flanking delimiter run - "emphasis_and_strong_emphasis_352", - "emphasis_and_strong_emphasis_367", - "emphasis_and_strong_emphasis_368", - "emphasis_and_strong_emphasis_372", - "emphasis_and_strong_emphasis_379", - "emphasis_and_strong_emphasis_388", - "emphasis_and_strong_emphasis_391", - "emphasis_and_strong_emphasis_406", - "emphasis_and_strong_emphasis_407", - "emphasis_and_strong_emphasis_408", - "emphasis_and_strong_emphasis_412", - "emphasis_and_strong_emphasis_413", - "emphasis_and_strong_emphasis_414", - "emphasis_and_strong_emphasis_416", - "emphasis_and_strong_emphasis_417", - "emphasis_and_strong_emphasis_418", - "emphasis_and_strong_emphasis_424", - "emphasis_and_strong_emphasis_425", - "emphasis_and_strong_emphasis_426", - "emphasis_and_strong_emphasis_429", - "emphasis_and_strong_emphasis_430", - "emphasis_and_strong_emphasis_431", - "emphasis_and_strong_emphasis_460", - "emphasis_and_strong_emphasis_467", - "emphasis_and_strong_emphasis_470", - "emphasis_and_strong_emphasis_471", - "emphasis_and_strong_emphasis_477", - "emphasis_and_strong_emphasis_478", -} - -for i in range(441, 447): - IGNORE_CASES.add("emphasis_and_strong_emphasis_" + str(i)) -for i in range(453, 459): - IGNORE_CASES.add("emphasis_and_strong_emphasis_" + str(i)) -for i in range(462, 466): - IGNORE_CASES.add("emphasis_and_strong_emphasis_" + str(i)) - - class TestCommonMark(BaseTestCase): @classmethod def ignore_case(cls, n): - return n in IGNORE_CASES or n in DIFF_CASES + return False def assert_case(self, n, text, html): result = mistune.html(text) Index: mistune-3.1.3/tests/test_misc.py =================================================================== --- mistune-3.1.3.orig/tests/test_misc.py +++ mistune-3.1.3/tests/test_misc.py @@ -26,6 +26,56 @@ class TestMiscCases(TestCase): result = md("foo\nbar") self.assertEqual(result.strip(), expected) + def test_speedup_plugin_is_compat_noop(self): + base = mistune.create_markdown(escape=False) + compat = mistune.create_markdown(escape=False, plugins=["speedup"]) + text = "foo **bar**\n\n[link](https://example.com)\n" + + self.assertEqual(compat(text), base(text)) + self.assertNotIn("paragraph", compat.block.rules) + self.assertNotIn("text", compat.inline.rules) + + def test_block_plain_paragraph_fast_path_interrupts(self): + cases = [ + ( + mistune.create_markdown(escape=False), + "Title\n---\n", + "

      Title

      ", + ), + ( + mistune.create_markdown(escape=False), + "foo\n- bar\n", + "

      foo

      \n
        \n
      • bar
        • \n
      ", + ), + ( + mistune.create_markdown(escape=False, plugins=["table"]), + "A | B\n--|--\n1 | 2\n", + "\n\n\n \n \n\n\n\n\n \n \n\n\n
      AB
      12
      ", + ), + ( + mistune.create_markdown(escape=False, plugins=["def_list"]), + "Term\n: def\n", + "
      \n
      Term
      \n
      def
      \n
      ", + ), + ] + for md, text, html in cases: + self.assertEqual(md(text).strip(), html) + + def test_block_quote_line_based_boundaries(self): + cases = { + "> foo\nbar\n": "
      \n

      foo\nbar

      \n
      ", + "> foo\n>\nbar\n": "
      \n

      foo

      \n
      \n

      bar

      ", + "> foo\n>\n> bar\n": "
      \n

      foo

      \n

      bar

      \n
      ", + "> foo\n---\n": "
      \n

      foo

      \n
      \n
      ", + } + for text, html in cases.items(): + self.assertEqual(mistune.html(text).strip(), html) + + def test_list_tab_continuation_columns(self): + result = mistune.html("-\t\tfoo\n") + expected = "
        \n
      •   foo
        \n
        • \n
      " + self.assertEqual(result.strip(), expected) + def test_escape_html(self): md = mistune.create_markdown(escape=True) result = md("
      1
      ") @@ -46,6 +96,16 @@ class TestMiscCases(TestCase): expected = '

      link

      ' self.assertEqual(result.strip(), expected) + def test_link_bracket_cache_cases(self): + cases = { + "[[a]](/url)": '

      [a]

      ', + "[a [b] c](/url)": '

      a [b] c

      ', + r"[a\]b](/url)": '

      a]b

      ', + "[a [b [c": "

      [a [b [c

      ", + } + for text, html in cases.items(): + self.assertEqual(mistune.html(text).strip(), html) + def test_allow_harmful_protocols(self): renderer = mistune.HTMLRenderer(allow_harmful_protocols=True) md = mistune.Markdown(renderer) @@ -104,6 +164,11 @@ class TestMiscCases(TestCase): ] self.assertEqual(result, expected) + def test_emphasis_default(self): + result = mistune.html("*_em_* __**strong**__") + expected = "

      em strong

      " + self.assertEqual(result.strip(), expected) + def test_ast_url(self): md = mistune.create_markdown(escape=False, renderer=None) label = 'hi &<>"' @@ -135,3 +200,13 @@ class TestMiscCases(TestCase): result = md("foo\n- bar\n\ntable") expected = "

      foo

      \n
        \n
      • bar
        • \n
      \n

      table

      " self.assertEqual(result.strip(), expected) + + def test_table_plugin_redos_candidates(self): + md = mistune.create_markdown(escape=False, plugins=["table"]) + md("|x" + " " * 16000 + "|\n|---|\n") + md("|x|\n" + "|" * 32000 + "\n") + + def test_def_list_plugin_redos_candidate(self): + md = mistune.create_markdown(escape=False, plugins=["def_list"]) + result = md("x\n" * 8000) + self.assertTrue(result.startswith("

      x\nx\n")) Index: mistune-3.1.3/tests/test_syntax.py =================================================================== --- mistune-3.1.3.orig/tests/test_syntax.py +++ mistune-3.1.3/tests/test_syntax.py @@ -9,4 +9,3 @@ class TestSyntax(BaseTestCase): TestSyntax.load_fixtures("fix-commonmark.txt") -TestSyntax.load_fixtures("diff-commonmark.txt")