|
53 | 53 | ) |
54 | 54 |
|
55 | 55 |
|
| 56 | +# A fenced code block delimiter: three or more backticks or tildes, optionally |
| 57 | +# indented by up to three spaces (CommonMark fence syntax). |
| 58 | +_CODE_FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})") |
| 59 | + |
| 60 | + |
| 61 | +def _is_code_fence(line: str) -> Optional[re.Match]: |
| 62 | + """Match a code fence delimiter at the start of a line, if any.""" |
| 63 | + return _CODE_FENCE_RE.match(line) |
| 64 | + |
| 65 | + |
| 66 | +def _closes_fence(line: str, fence_char: str, fence_len: int) -> bool: |
| 67 | + """Whether a line inside a fenced code block closes the fence.""" |
| 68 | + match = _is_code_fence(line) |
| 69 | + if match is None: |
| 70 | + return False |
| 71 | + marker = match.group(1) |
| 72 | + # The closing fence uses the same character, is at least as long, and |
| 73 | + # carries no info string. |
| 74 | + return marker[0] == fence_char and len(marker) >= fence_len and not line[match.end() :].strip() |
| 75 | + |
| 76 | + |
| 77 | +def _normalize_whitespace_outside_code_fences(text: str) -> str: |
| 78 | + """Normalize whitespace while leaving fenced code blocks untouched. |
| 79 | +
|
| 80 | + Outside code fences, the previous normalization is applied: trailing |
| 81 | + whitespace is stripped from each line and runs of three or more newlines |
| 82 | + are collapsed to two. Content inside a fenced code block is |
| 83 | + whitespace-significant, so it is passed through verbatim apart from |
| 84 | + CRLF newline normalization. |
| 85 | + """ |
| 86 | + lines = re.split(r"\r?\n", text) |
| 87 | + out: list[str] = [] |
| 88 | + segment: list[str] = [] |
| 89 | + fence_char = "" |
| 90 | + fence_len = 0 |
| 91 | + |
| 92 | + def flush_segment(fence_follows: bool) -> None: |
| 93 | + if segment: |
| 94 | + normalized = re.sub( |
| 95 | + r"\n{3,}", "\n\n", "\n".join(line.rstrip() for line in segment) |
| 96 | + ) |
| 97 | + if fence_follows: |
| 98 | + # The final "\n".join(out) adds one newline before the fence |
| 99 | + # line, so trailing newlines here would stack to three. |
| 100 | + normalized = normalized.rstrip("\n") |
| 101 | + out.append(normalized) |
| 102 | + segment.clear() |
| 103 | + |
| 104 | + for line in lines: |
| 105 | + if fence_char: |
| 106 | + # Inside a code fence: keep the line exactly as it is. |
| 107 | + out.append(line) |
| 108 | + if _closes_fence(line, fence_char, fence_len): |
| 109 | + fence_char = "" |
| 110 | + continue |
| 111 | + |
| 112 | + match = _is_code_fence(line) |
| 113 | + if match is not None: |
| 114 | + flushed_prose = bool(segment) |
| 115 | + flush_segment(fence_follows=True) |
| 116 | + # Keep exactly one blank line between preceding prose and a fence. |
| 117 | + if flushed_prose: |
| 118 | + out.append("") |
| 119 | + # An opening fence may carry an info string (e.g. ```python). |
| 120 | + marker = match.group(1) |
| 121 | + fence_char = marker[0] |
| 122 | + fence_len = len(marker) |
| 123 | + out.append(line.rstrip()) |
| 124 | + continue |
| 125 | + |
| 126 | + segment.append(line) |
| 127 | + |
| 128 | + flush_segment(fence_follows=False) |
| 129 | + return "\n".join(out) |
| 130 | + |
| 131 | + |
56 | 132 | def _get_content_disposition_filename(content_disposition: str) -> Optional[str]: |
57 | 133 | message = Message() |
58 | 134 | message["content-disposition"] = content_disposition |
@@ -683,10 +759,9 @@ def _convert( |
683 | 759 |
|
684 | 760 | if res is not None: |
685 | 761 | # Normalize the content |
686 | | - res.text_content = "\n".join( |
687 | | - [line.rstrip() for line in re.split(r"\r?\n", res.text_content)] |
| 762 | + res.text_content = _normalize_whitespace_outside_code_fences( |
| 763 | + res.text_content |
688 | 764 | ) |
689 | | - res.text_content = re.sub(r"\n{3,}", "\n\n", res.text_content) |
690 | 765 | return res |
691 | 766 |
|
692 | 767 | # If we got this far without success, report any exceptions |
|
0 commit comments