From f91f860c03b9c945381c78eaf6f438be0449dd15 Mon Sep 17 00:00:00 2001 From: Deepak Ganesh Date: Sat, 25 Jul 2026 16:09:39 +0530 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20Preserve=20parsed-lit?= =?UTF-8?q?eral=20whitespace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Protect line-edge whitespace while Markdown inline markup is parsed, then restore it in the generated text nodes. Add regression coverage for spaces, tabs, trailing whitespace, inline markup, and private-use characters. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 6 ++++ myst_parser/mdit_to_docutils/base.py | 10 +++++- myst_parser/mocking.py | 52 ++++++++++++++++++++++++++-- tests/test_docutils.py | 21 +++++++++++ 4 files changed, 86 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff2d904f..8e86fbd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +### 🐛 Bug Fixes + +- 🐛 Preserve leading whitespace in `parsed-literal` directive content. + ## 5.1.0 - 2026-05-13 ### ✨ New Features diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index 68cfae72..b7d4ee2d 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -32,6 +32,7 @@ from docutils.languages import get_language from docutils.parsers.rst import Directive, DirectiveError, directives, roles from docutils.parsers.rst import Parser as RSTParser +from docutils.parsers.rst.directives.body import ParsedLiteral from docutils.parsers.rst.directives.misc import Include from docutils.parsers.rst.languages import get_language as get_language_rst from docutils.statemachine import StringList @@ -1898,7 +1899,14 @@ def run_directive( ) else: state_machine = MockStateMachine(self, position) - state = MockState(self, state_machine, position) + state = MockState( + self, + state_machine, + position, + preserve_inline_edge_whitespace=issubclass( + directive_class, ParsedLiteral + ), + ) directive_instance = directive_class( name=name, # the list of positional arguments diff --git a/myst_parser/mocking.py b/myst_parser/mocking.py index b65888e7..e536071b 100644 --- a/myst_parser/mocking.py +++ b/myst_parser/mocking.py @@ -61,7 +61,13 @@ def problematic( return problematic def parse( - self, text: str, lineno: int, memo: Any, parent: nodes.Node + self, + text: str, + lineno: int, + memo: Any, + parent: nodes.Node, + *, + preserve_edge_whitespace: bool = False, ) -> tuple[list[nodes.Node], list[nodes.system_message]]: """Parse the text and return a list of nodes.""" # note the only place this is normally called, @@ -70,6 +76,18 @@ def parse( # self.reporter = memo.reporter # self.document = memo.document # self.language = memo.language + whitespace_markers: tuple[str, str] | None = None + if preserve_edge_whitespace: + whitespace_markers = self._unused_private_chars(text) + space_marker, tab_marker = whitespace_markers + text = re.sub( + r"(?m)(?:^[ \t]+|[ \t]+$)", + lambda match: ( + match.group().replace(" ", space_marker).replace("\t", tab_marker) + ), + text, + ) + with self._renderer.current_node_context(parent): # the parent is never actually appended to though, # so we make a temporary parent to parse into @@ -77,8 +95,29 @@ def parse( with self._renderer.current_node_context(container): self._renderer.nested_render_text(text, lineno, inline=True) + if whitespace_markers: + space_marker, tab_marker = whitespace_markers + for text_node in list(container.findall(nodes.Text)): + restored = ( + str(text_node).replace(space_marker, " ").replace(tab_marker, "\t") + ) + if restored != str(text_node): + text_node.parent.replace(text_node, nodes.Text(restored)) + return container.children, [] + @staticmethod + def _unused_private_chars(text: str) -> tuple[str, str]: + """Return two private-use characters that do not occur in ``text``.""" + markers: list[str] = [] + for codepoint in range(0xE000, 0xF900): + marker = chr(codepoint) + if marker not in text: + markers.append(marker) + if len(markers) == 2: + return markers[0], markers[1] + raise ValueError("Unable to reserve private-use whitespace markers") + def __getattr__(self, name: str): """This method is only be called if the attribute requested has not been defined. Defined attributes will not be overridden. @@ -104,9 +143,12 @@ def __init__( renderer: DocutilsRenderer, state_machine: MockStateMachine, lineno: int, + *, + preserve_inline_edge_whitespace: bool = False, ): self._renderer = renderer self._lineno = lineno + self._preserve_inline_edge_whitespace = preserve_inline_edge_whitespace self.document = renderer.document self.reporter = renderer.document.reporter self.state_machine = state_machine @@ -197,7 +239,13 @@ def inline_text( :returns: (list of nodes, list of messages) """ - return self.inliner.parse(text, lineno, self.memo, self._renderer.current_node) + return self.inliner.parse( + text, + lineno, + self.memo, + self._renderer.current_node, + preserve_edge_whitespace=self._preserve_inline_edge_whitespace, + ) # U+2014 is an em-dash: attribution_pattern = re.compile("^((?:---?(?!-)|\u2014) *)(.+)") diff --git a/tests/test_docutils.py b/tests/test_docutils.py index 890badbc..c766d1fc 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -48,6 +48,27 @@ def test_parser(): ) +def test_parsed_literal_preserves_leading_whitespace(): + """Line-edge whitespace survives inline parsing in parsed literals.""" + source = "\n".join( + [ + "```{parsed-literal}", + "123", + " 23", + "", + "\t**3** ", + "\ue000 marker", + "```", + "", + ] + ) + document = publish_doctree(source, parser=Parser()) + + literal = next(document.findall(nodes.literal_block)) + assert literal.astext() == "123\n 23\n\n\t3 \n\ue000 marker" + assert [node.astext() for node in literal.findall(nodes.strong)] == ["3"] + + def test_cli_html(monkeypatch, capsys): monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) cli_html([]) From fc8d794b834d99bc0b21f6ac43bc1c821a0a5fea Mon Sep 17 00:00:00 2001 From: Deepak Ganesh Date: Sat, 25 Jul 2026 16:49:09 +0530 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20Preserve=20whitespace?= =?UTF-8?q?=20context=20in=20parsed=20literals?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace private-use sentinels with an opt-in markdown-it newline rule that retains adjacent whitespace without changing delimiter classification. Add regressions for cross-line emphasis, decoded private-use entities, and the full private-use range. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 +- myst_parser/mocking.py | 73 +++++++++++++++++++++--------------------- tests/test_docutils.py | 32 +++++++++++++++++- 3 files changed, 68 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e86fbd7..e179d07e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### 🐛 Bug Fixes -- 🐛 Preserve leading whitespace in `parsed-literal` directive content. +- 🐛 Preserve line-edge whitespace in `parsed-literal` directive content. ## 5.1.0 - 2026-05-13 diff --git a/myst_parser/mocking.py b/myst_parser/mocking.py index e536071b..496a133a 100644 --- a/myst_parser/mocking.py +++ b/myst_parser/mocking.py @@ -17,6 +17,7 @@ from docutils.parsers.rst.states import Body, Inliner, RSTStateMachine from docutils.statemachine import StringList from docutils.utils import unescape +from markdown_it.rules_inline import StateInline from .parsers.directives import MarkupError, parse_directive_text @@ -24,6 +25,22 @@ from .mdit_to_docutils.base import DocutilsRenderer +_PRESERVE_LINE_EDGE_WHITESPACE = "myst_preserve_line_edge_whitespace" +_PRESERVE_NEWLINE_RULE = "myst_preserve_newline" + + +def _preserve_newline(state: StateInline, silent: bool) -> bool: + """Render a newline without consuming adjacent whitespace.""" + if not state.env.get(_PRESERVE_LINE_EDGE_WHITESPACE): + return False + if state.src[state.pos] != "\n": + return False + if not silent: + state.push("softbreak", "br", 0) + state.pos += 1 + return True + + class MockingError(Exception): """An exception to signal an error during mocking of docutils components.""" @@ -37,6 +54,10 @@ class MockInliner: def __init__(self, renderer: DocutilsRenderer): """Initialize the mock inliner.""" self._renderer = renderer + if _PRESERVE_NEWLINE_RULE not in renderer.md.inline.ruler.get_all_rules(): + renderer.md.inline.ruler.before( + "newline", _PRESERVE_NEWLINE_RULE, _preserve_newline + ) # here we mock that the `parse` method has already been called # which is where these attributes are set (via the RST state Memo) self.document = renderer.document @@ -76,48 +97,26 @@ def parse( # self.reporter = memo.reporter # self.document = memo.document # self.language = memo.language - whitespace_markers: tuple[str, str] | None = None + previous_preserve = self._renderer.md_env.get(_PRESERVE_LINE_EDGE_WHITESPACE) if preserve_edge_whitespace: - whitespace_markers = self._unused_private_chars(text) - space_marker, tab_marker = whitespace_markers - text = re.sub( - r"(?m)(?:^[ \t]+|[ \t]+$)", - lambda match: ( - match.group().replace(" ", space_marker).replace("\t", tab_marker) - ), - text, - ) - - with self._renderer.current_node_context(parent): - # the parent is never actually appended to though, - # so we make a temporary parent to parse into - container = nodes.Element() - with self._renderer.current_node_context(container): - self._renderer.nested_render_text(text, lineno, inline=True) - - if whitespace_markers: - space_marker, tab_marker = whitespace_markers - for text_node in list(container.findall(nodes.Text)): - restored = ( - str(text_node).replace(space_marker, " ").replace(tab_marker, "\t") + self._renderer.md_env[_PRESERVE_LINE_EDGE_WHITESPACE] = True + try: + with self._renderer.current_node_context(parent): + # the parent is never actually appended to though, + # so we make a temporary parent to parse into + container = nodes.Element() + with self._renderer.current_node_context(container): + self._renderer.nested_render_text(text, lineno, inline=True) + finally: + if previous_preserve is None: + self._renderer.md_env.pop(_PRESERVE_LINE_EDGE_WHITESPACE, None) + else: + self._renderer.md_env[_PRESERVE_LINE_EDGE_WHITESPACE] = ( + previous_preserve ) - if restored != str(text_node): - text_node.parent.replace(text_node, nodes.Text(restored)) return container.children, [] - @staticmethod - def _unused_private_chars(text: str) -> tuple[str, str]: - """Return two private-use characters that do not occur in ``text``.""" - markers: list[str] = [] - for codepoint in range(0xE000, 0xF900): - marker = chr(codepoint) - if marker not in text: - markers.append(marker) - if len(markers) == 2: - return markers[0], markers[1] - raise ValueError("Unable to reserve private-use whitespace markers") - def __getattr__(self, name: str): """This method is only be called if the attribute requested has not been defined. Defined attributes will not be overridden. diff --git a/tests/test_docutils.py b/tests/test_docutils.py index c766d1fc..1f0b8699 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -48,7 +48,7 @@ def test_parser(): ) -def test_parsed_literal_preserves_leading_whitespace(): +def test_parsed_literal_preserves_line_edge_whitespace(): """Line-edge whitespace survives inline parsing in parsed literals.""" source = "\n".join( [ @@ -69,6 +69,36 @@ def test_parsed_literal_preserves_leading_whitespace(): assert [node.astext() for node in literal.findall(nodes.strong)] == ["3"] +def test_parsed_literal_whitespace_preserves_delimiter_boundaries(): + """Whitespace remains visible to CommonMark delimiter classification.""" + source = "\n".join(["```{parsed-literal}", "foo * ", "bar*", "```", ""]) + document = publish_doctree(source, parser=Parser()) + + literal = next(document.findall(nodes.literal_block)) + assert literal.astext() == "foo * \nbar*" + assert not list(literal.findall(nodes.emphasis)) + + +def test_parsed_literal_encoded_private_use_entities(): + """Decoded private-use entities are not mistaken for whitespace markers.""" + document = publish_doctree( + "```{parsed-literal}\n\n \n```\n", parser=Parser() + ) + + literal = next(document.findall(nodes.literal_block)) + assert literal.astext() == "\ue000\n \ue001" + + +def test_parsed_literal_all_private_use_characters(): + """Using every private-use character cannot exhaust internal markers.""" + private_use = "".join(chr(codepoint) for codepoint in range(0xE000, 0xF900)) + source = f"```{{parsed-literal}}\n{private_use}\n indented\n```\n" + document = publish_doctree(source, parser=Parser()) + + literal = next(document.findall(nodes.literal_block)) + assert literal.astext() == f"{private_use}\n indented" + + def test_cli_html(monkeypatch, capsys): monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) cli_html([])