|
| 1 | +# The page title for the fetch server's simplified (Markdown) result (#2472). |
| 2 | +# |
| 3 | +# Readability.js removes an article heading that repeats the document title, |
| 4 | +# and neither extractor returns the `<title>` itself in the content the server |
| 5 | +# converts, so a page such as a Sphinx release note came back without the line |
| 6 | +# that says what it is. This module reads the title from the raw HTML and puts |
| 7 | +# it at the top of the Markdown as a level-one heading, unless the Markdown |
| 8 | +# already opens with it. |
| 9 | +# |
| 10 | +# It works from the raw page rather than from readabilipy's result, so it is |
| 11 | +# independent of how the extractor is invoked and is applied at the one call |
| 12 | +# site in `fetch_url` that simplifies HTML. The raw result (`raw: true`) and |
| 13 | +# non-HTML bodies never pass through it. |
| 14 | + |
| 15 | +import re |
| 16 | + |
| 17 | +from bs4 import BeautifulSoup, Tag |
| 18 | + |
| 19 | +# The marker `extract_content_from_html` returns when a page simplifies to |
| 20 | +# nothing. A title on top of it would make the failure read as content. |
| 21 | +_FAILURE_PREFIX = "<error>" |
| 22 | + |
| 23 | +# Titles are compared word by word: case, Markdown heading markers, emphasis, |
| 24 | +# escapes, Sphinx's "¶" permalink and other punctuation are ignored, but word |
| 25 | +# boundaries are kept, so "Rust" does not match "Trust". |
| 26 | +_WORD = re.compile(r"[^\W_]+") |
| 27 | +# Characters that Markdown or inline HTML would interpret in the emitted |
| 28 | +# heading ("<T>" read as a tag, "__init__" as emphasis, "<" as an entity), |
| 29 | +# each escaped with a backslash so the heading shows the title as written. |
| 30 | +_MARKDOWN_SPECIAL = re.compile(r"([\\`*_\[\]<>#&|~])") |
| 31 | + |
| 32 | + |
| 33 | +def _normalized(text: str) -> str: |
| 34 | + """``text`` as its casefolded words, one space apart, padded with spaces. |
| 35 | +
|
| 36 | + The padding makes ``in`` a whole-word, whole-phrase containment test. |
| 37 | + """ |
| 38 | + words = _WORD.findall(text.casefold()) |
| 39 | + return f" {' '.join(words)} " if words else "" |
| 40 | + |
| 41 | + |
| 42 | +def _starts_tag(char: str) -> bool: |
| 43 | + return char.isascii() and (char.isalpha() or char in "/!?") |
| 44 | + |
| 45 | + |
| 46 | +def _visible(line: str) -> str: |
| 47 | + """The text a reader sees on one Markdown line. |
| 48 | +
|
| 49 | + A link's destination (`](...)`) and an HTML tag (an autolink's URL |
| 50 | + included) are dropped; a code span's contents are kept, tag-like text |
| 51 | + and all. As in CommonMark, a `<` starts a tag only before a letter, `/`, |
| 52 | + `!` or `?` (so "x < 5 and y > 3" is text), and a backslash-escaped |
| 53 | + character is text. One pass, so a long line with an unterminated `](`, `<` or |
| 54 | + backtick costs linear time: each closer is searched for forward from |
| 55 | + where it is needed, and once a search for one fails it is never repeated. |
| 56 | + """ |
| 57 | + out: list[str] = [] |
| 58 | + missing: set[str] = set() |
| 59 | + i, end = 0, len(line) |
| 60 | + while i < end: |
| 61 | + if line[i] == "\\" and i + 1 < end: |
| 62 | + out.append(line[i + 1]) |
| 63 | + i += 2 |
| 64 | + continue |
| 65 | + for opener, closer, keep in ( |
| 66 | + ("`", "`", True), |
| 67 | + ("<", ">", False), |
| 68 | + ("](", ")", False), |
| 69 | + ): |
| 70 | + if closer in missing or not line.startswith(opener, i): |
| 71 | + continue |
| 72 | + if opener == "<" and not _starts_tag(line[i + 1 : i + 2]): |
| 73 | + continue |
| 74 | + j = line.find(closer, i + len(opener)) |
| 75 | + if j == -1: |
| 76 | + missing.add(closer) |
| 77 | + continue |
| 78 | + out.append(line[i + len(opener) : j] if keep else " ") |
| 79 | + i = j + 1 |
| 80 | + break |
| 81 | + else: |
| 82 | + out.append(line[i]) |
| 83 | + i += 1 |
| 84 | + return "".join(out) |
| 85 | + |
| 86 | + |
| 87 | +def _collapsed(text: str) -> str: |
| 88 | + return " ".join(text.split()) |
| 89 | + |
| 90 | + |
| 91 | +def _text(candidate: str) -> str: |
| 92 | + # A trailing "¶" is a permalink that is not marked up as one. |
| 93 | + return _collapsed(candidate).rstrip("¶").strip() |
| 94 | + |
| 95 | + |
| 96 | +def page_title(html: str) -> str | None: |
| 97 | + """The page's title, or None when the page names none. |
| 98 | +
|
| 99 | + In order: `og:title`; the first `<h1>` when `<title>` holds it, since |
| 100 | + `<title>` then only adds the site's name ("What's new — pandas docs"); |
| 101 | + `<title>`; the first `<h1>`. |
| 102 | + """ |
| 103 | + soup = BeautifulSoup(html, "html.parser") |
| 104 | + og = soup.find("meta", attrs={"property": "og:title"}) |
| 105 | + og_title = _text(str(og.get("content", ""))) if isinstance(og, Tag) else "" |
| 106 | + title = _text(soup.title.get_text(" ")) if soup.title else "" |
| 107 | + h1 = soup.find("h1") |
| 108 | + heading = "" |
| 109 | + if isinstance(h1, Tag): |
| 110 | + # Sphinx appends a permalink anchor ("¶", or "#" in some themes) to |
| 111 | + # each heading; it is not part of the heading's text. |
| 112 | + for link in h1.find_all("a", class_="headerlink"): |
| 113 | + link.decompose() |
| 114 | + heading = _text(h1.get_text(" ")) |
| 115 | + heading_in_title = bool(_normalized(heading)) and _normalized( |
| 116 | + heading |
| 117 | + ) in _normalized(title) |
| 118 | + if og_title: |
| 119 | + return og_title |
| 120 | + if heading_in_title: |
| 121 | + return heading |
| 122 | + return title or heading or None |
| 123 | + |
| 124 | + |
| 125 | +def _opens_with_title(content: str, title: str) -> bool: |
| 126 | + """Whether the Markdown's first non-empty line already carries the title. |
| 127 | +
|
| 128 | + A first line holding the whole title counts (the pure-Python extractor |
| 129 | + opens with the title in a paragraph), and so does a first heading that the |
| 130 | + title holds: `<title>` often adds the site name ("What's new — pandas |
| 131 | + docs") to the page's own heading ("What's new"). |
| 132 | + """ |
| 133 | + first = next((line for line in content.splitlines() if line.strip()), "") |
| 134 | + line, wanted = _normalized(_visible(first)), _normalized(title) |
| 135 | + if not line or not wanted: |
| 136 | + return False |
| 137 | + if wanted in line: |
| 138 | + return True |
| 139 | + return first.lstrip().startswith("#") and line in wanted |
| 140 | + |
| 141 | + |
| 142 | +def with_page_title(html: str, content: str) -> str: |
| 143 | + """``content`` with the page's title as a heading on top, if it lacks one.""" |
| 144 | + if content.startswith(_FAILURE_PREFIX): |
| 145 | + return content |
| 146 | + title = page_title(html) |
| 147 | + if title is None or _opens_with_title(content, title): |
| 148 | + return content |
| 149 | + heading = _MARKDOWN_SPECIAL.sub(r"\\\1", title) |
| 150 | + return f"# {heading}\n\n{content.lstrip()}" |
0 commit comments