diff --git a/CHANGELOG.md b/CHANGELOG.md index ff2d904f..e179d07e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +### 🐛 Bug Fixes + +- 🐛 Preserve line-edge whitespace in `parsed-literal` directive content. + ## 5.1.0 - 2026-05-13 ### ✨ New Features diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index 68cfae72..b7d4ee2d 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -32,6 +32,7 @@ from docutils.languages import get_language from docutils.parsers.rst import Directive, DirectiveError, directives, roles from docutils.parsers.rst import Parser as RSTParser +from docutils.parsers.rst.directives.body import ParsedLiteral from docutils.parsers.rst.directives.misc import Include from docutils.parsers.rst.languages import get_language as get_language_rst from docutils.statemachine import StringList @@ -1898,7 +1899,14 @@ def run_directive( ) else: state_machine = MockStateMachine(self, position) - state = MockState(self, state_machine, position) + state = MockState( + self, + state_machine, + position, + preserve_inline_edge_whitespace=issubclass( + directive_class, ParsedLiteral + ), + ) directive_instance = directive_class( name=name, # the list of positional arguments diff --git a/myst_parser/mocking.py b/myst_parser/mocking.py index b65888e7..496a133a 100644 --- a/myst_parser/mocking.py +++ b/myst_parser/mocking.py @@ -17,6 +17,7 @@ from docutils.parsers.rst.states import Body, Inliner, RSTStateMachine from docutils.statemachine import StringList from docutils.utils import unescape +from markdown_it.rules_inline import StateInline from .parsers.directives import MarkupError, parse_directive_text @@ -24,6 +25,22 @@ from .mdit_to_docutils.base import DocutilsRenderer +_PRESERVE_LINE_EDGE_WHITESPACE = "myst_preserve_line_edge_whitespace" +_PRESERVE_NEWLINE_RULE = "myst_preserve_newline" + + +def _preserve_newline(state: StateInline, silent: bool) -> bool: + """Render a newline without consuming adjacent whitespace.""" + if not state.env.get(_PRESERVE_LINE_EDGE_WHITESPACE): + return False + if state.src[state.pos] != "\n": + return False + if not silent: + state.push("softbreak", "br", 0) + state.pos += 1 + return True + + class MockingError(Exception): """An exception to signal an error during mocking of docutils components.""" @@ -37,6 +54,10 @@ class MockInliner: def __init__(self, renderer: DocutilsRenderer): """Initialize the mock inliner.""" self._renderer = renderer + if _PRESERVE_NEWLINE_RULE not in renderer.md.inline.ruler.get_all_rules(): + renderer.md.inline.ruler.before( + "newline", _PRESERVE_NEWLINE_RULE, _preserve_newline + ) # here we mock that the `parse` method has already been called # which is where these attributes are set (via the RST state Memo) self.document = renderer.document @@ -61,7 +82,13 @@ def problematic( return problematic def parse( - self, text: str, lineno: int, memo: Any, parent: nodes.Node + self, + text: str, + lineno: int, + memo: Any, + parent: nodes.Node, + *, + preserve_edge_whitespace: bool = False, ) -> tuple[list[nodes.Node], list[nodes.system_message]]: """Parse the text and return a list of nodes.""" # note the only place this is normally called, @@ -70,12 +97,23 @@ def parse( # self.reporter = memo.reporter # self.document = memo.document # self.language = memo.language - with self._renderer.current_node_context(parent): - # the parent is never actually appended to though, - # so we make a temporary parent to parse into - container = nodes.Element() - with self._renderer.current_node_context(container): - self._renderer.nested_render_text(text, lineno, inline=True) + previous_preserve = self._renderer.md_env.get(_PRESERVE_LINE_EDGE_WHITESPACE) + if preserve_edge_whitespace: + self._renderer.md_env[_PRESERVE_LINE_EDGE_WHITESPACE] = True + try: + with self._renderer.current_node_context(parent): + # the parent is never actually appended to though, + # so we make a temporary parent to parse into + container = nodes.Element() + with self._renderer.current_node_context(container): + self._renderer.nested_render_text(text, lineno, inline=True) + finally: + if previous_preserve is None: + self._renderer.md_env.pop(_PRESERVE_LINE_EDGE_WHITESPACE, None) + else: + self._renderer.md_env[_PRESERVE_LINE_EDGE_WHITESPACE] = ( + previous_preserve + ) return container.children, [] @@ -104,9 +142,12 @@ def __init__( renderer: DocutilsRenderer, state_machine: MockStateMachine, lineno: int, + *, + preserve_inline_edge_whitespace: bool = False, ): self._renderer = renderer self._lineno = lineno + self._preserve_inline_edge_whitespace = preserve_inline_edge_whitespace self.document = renderer.document self.reporter = renderer.document.reporter self.state_machine = state_machine @@ -197,7 +238,13 @@ def inline_text( :returns: (list of nodes, list of messages) """ - return self.inliner.parse(text, lineno, self.memo, self._renderer.current_node) + return self.inliner.parse( + text, + lineno, + self.memo, + self._renderer.current_node, + preserve_edge_whitespace=self._preserve_inline_edge_whitespace, + ) # U+2014 is an em-dash: attribution_pattern = re.compile("^((?:---?(?!-)|\u2014) *)(.+)") diff --git a/tests/test_docutils.py b/tests/test_docutils.py index 890badbc..1f0b8699 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -48,6 +48,57 @@ def test_parser(): ) +def test_parsed_literal_preserves_line_edge_whitespace(): + """Line-edge whitespace survives inline parsing in parsed literals.""" + source = "\n".join( + [ + "```{parsed-literal}", + "123", + " 23", + "", + "\t**3** ", + "\ue000 marker", + "```", + "", + ] + ) + document = publish_doctree(source, parser=Parser()) + + literal = next(document.findall(nodes.literal_block)) + assert literal.astext() == "123\n 23\n\n\t3 \n\ue000 marker" + assert [node.astext() for node in literal.findall(nodes.strong)] == ["3"] + + +def test_parsed_literal_whitespace_preserves_delimiter_boundaries(): + """Whitespace remains visible to CommonMark delimiter classification.""" + source = "\n".join(["```{parsed-literal}", "foo * ", "bar*", "```", ""]) + document = publish_doctree(source, parser=Parser()) + + literal = next(document.findall(nodes.literal_block)) + assert literal.astext() == "foo * \nbar*" + assert not list(literal.findall(nodes.emphasis)) + + +def test_parsed_literal_encoded_private_use_entities(): + """Decoded private-use entities are not mistaken for whitespace markers.""" + document = publish_doctree( + "```{parsed-literal}\n\n \n```\n", parser=Parser() + ) + + literal = next(document.findall(nodes.literal_block)) + assert literal.astext() == "\ue000\n \ue001" + + +def test_parsed_literal_all_private_use_characters(): + """Using every private-use character cannot exhaust internal markers.""" + private_use = "".join(chr(codepoint) for codepoint in range(0xE000, 0xF900)) + source = f"```{{parsed-literal}}\n{private_use}\n indented\n```\n" + document = publish_doctree(source, parser=Parser()) + + literal = next(document.findall(nodes.literal_block)) + assert literal.astext() == f"{private_use}\n indented" + + def test_cli_html(monkeypatch, capsys): monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) cli_html([])