Skip to content

Commit b6bb2f6

Browse files
authored
Merge pull request #5398 from modelcontextprotocol/v2/feat/2472-fetch-page-title
feat(fetch): open the simplified result with the page title
2 parents 4aac5e4 + 8b836f7 commit b6bb2f6

4 files changed

Lines changed: 380 additions & 1 deletion

File tree

‎src/fetch/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ Requires MCP Python SDK 2.x (`mcp>=2.3,<3`).
1515
1616
The fetch tool will truncate the response, but by using the `start_index` argument, you can specify where to start the content extraction. This lets models read a webpage in chunks, until they find the information they need.
1717

18+
The markdown of an HTML page opens with the page's title as a `#` heading (its `og:title`, else its `<h1>` when the `<title>` holds it, else its `<title>` or `<h1>`), because the extraction drops a heading that repeats the title. It is not added when the markdown already opens with it, to a raw result, or to a page that failed to simplify.
19+
1820
### Available Tools
1921

2022
- `fetch` - Fetches a URL from the internet and extracts its contents as markdown.

‎src/fetch/src/mcp_server_fetch/server.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141
from protego import Protego
4242
from pydantic import AnyUrl, Field
4343

44+
from .title import with_page_title
45+
4446
# The version this server reports in serverInfo, read from the installed
4547
# distribution's metadata (pyproject.toml) so it cannot drift from the
4648
# published version (#360). Without it the SDK reports its own `mcp` version.
@@ -601,7 +603,7 @@ async def _check_redirect_robots_txt(request: httpx.Request) -> None:
601603
content = await asyncio.to_thread(
602604
extract_content_from_html, page_raw, use_readability_js
603605
)
604-
return content, ""
606+
return await asyncio.to_thread(with_page_title, page_raw, content), ""
605607

606608
# A response with no Content-Type would otherwise leave an empty slot here.
607609
declared_type = content_type or "(not declared)"
Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
# The page title for the fetch server's simplified (Markdown) result (#2472).
2+
#
3+
# Readability.js removes an article heading that repeats the document title,
4+
# and neither extractor returns the `<title>` itself in the content the server
5+
# converts, so a page such as a Sphinx release note came back without the line
6+
# that says what it is. This module reads the title from the raw HTML and puts
7+
# it at the top of the Markdown as a level-one heading, unless the Markdown
8+
# already opens with it.
9+
#
10+
# It works from the raw page rather than from readabilipy's result, so it is
11+
# independent of how the extractor is invoked and is applied at the one call
12+
# site in `fetch_url` that simplifies HTML. The raw result (`raw: true`) and
13+
# non-HTML bodies never pass through it.
14+
15+
import re
16+
17+
from bs4 import BeautifulSoup, Tag
18+
19+
# The marker `extract_content_from_html` returns when a page simplifies to
20+
# nothing. A title on top of it would make the failure read as content.
21+
_FAILURE_PREFIX = "<error>"
22+
23+
# Titles are compared word by word: case, Markdown heading markers, emphasis,
24+
# escapes, Sphinx's "¶" permalink and other punctuation are ignored, but word
25+
# boundaries are kept, so "Rust" does not match "Trust".
26+
_WORD = re.compile(r"[^\W_]+")
27+
# Characters that Markdown or inline HTML would interpret in the emitted
28+
# heading ("<T>" read as a tag, "__init__" as emphasis, "&lt;" as an entity),
29+
# each escaped with a backslash so the heading shows the title as written.
30+
_MARKDOWN_SPECIAL = re.compile(r"([\\`*_\[\]<>#&|~])")
31+
32+
33+
def _normalized(text: str) -> str:
34+
"""``text`` as its casefolded words, one space apart, padded with spaces.
35+
36+
The padding makes ``in`` a whole-word, whole-phrase containment test.
37+
"""
38+
words = _WORD.findall(text.casefold())
39+
return f" {' '.join(words)} " if words else ""
40+
41+
42+
def _starts_tag(char: str) -> bool:
43+
return char.isascii() and (char.isalpha() or char in "/!?")
44+
45+
46+
def _visible(line: str) -> str:
47+
"""The text a reader sees on one Markdown line.
48+
49+
A link's destination (`](...)`) and an HTML tag (an autolink's URL
50+
included) are dropped; a code span's contents are kept, tag-like text
51+
and all. As in CommonMark, a `<` starts a tag only before a letter, `/`,
52+
`!` or `?` (so "x < 5 and y > 3" is text), and a backslash-escaped
53+
character is text. One pass, so a long line with an unterminated `](`, `<` or
54+
backtick costs linear time: each closer is searched for forward from
55+
where it is needed, and once a search for one fails it is never repeated.
56+
"""
57+
out: list[str] = []
58+
missing: set[str] = set()
59+
i, end = 0, len(line)
60+
while i < end:
61+
if line[i] == "\\" and i + 1 < end:
62+
out.append(line[i + 1])
63+
i += 2
64+
continue
65+
for opener, closer, keep in (
66+
("`", "`", True),
67+
("<", ">", False),
68+
("](", ")", False),
69+
):
70+
if closer in missing or not line.startswith(opener, i):
71+
continue
72+
if opener == "<" and not _starts_tag(line[i + 1 : i + 2]):
73+
continue
74+
j = line.find(closer, i + len(opener))
75+
if j == -1:
76+
missing.add(closer)
77+
continue
78+
out.append(line[i + len(opener) : j] if keep else " ")
79+
i = j + 1
80+
break
81+
else:
82+
out.append(line[i])
83+
i += 1
84+
return "".join(out)
85+
86+
87+
def _collapsed(text: str) -> str:
88+
return " ".join(text.split())
89+
90+
91+
def _text(candidate: str) -> str:
92+
# A trailing "¶" is a permalink that is not marked up as one.
93+
return _collapsed(candidate).rstrip("¶").strip()
94+
95+
96+
def page_title(html: str) -> str | None:
97+
"""The page's title, or None when the page names none.
98+
99+
In order: `og:title`; the first `<h1>` when `<title>` holds it, since
100+
`<title>` then only adds the site's name ("What's new — pandas docs");
101+
`<title>`; the first `<h1>`.
102+
"""
103+
soup = BeautifulSoup(html, "html.parser")
104+
og = soup.find("meta", attrs={"property": "og:title"})
105+
og_title = _text(str(og.get("content", ""))) if isinstance(og, Tag) else ""
106+
title = _text(soup.title.get_text(" ")) if soup.title else ""
107+
h1 = soup.find("h1")
108+
heading = ""
109+
if isinstance(h1, Tag):
110+
# Sphinx appends a permalink anchor ("¶", or "#" in some themes) to
111+
# each heading; it is not part of the heading's text.
112+
for link in h1.find_all("a", class_="headerlink"):
113+
link.decompose()
114+
heading = _text(h1.get_text(" "))
115+
heading_in_title = bool(_normalized(heading)) and _normalized(
116+
heading
117+
) in _normalized(title)
118+
if og_title:
119+
return og_title
120+
if heading_in_title:
121+
return heading
122+
return title or heading or None
123+
124+
125+
def _opens_with_title(content: str, title: str) -> bool:
126+
"""Whether the Markdown's first non-empty line already carries the title.
127+
128+
A first line holding the whole title counts (the pure-Python extractor
129+
opens with the title in a paragraph), and so does a first heading that the
130+
title holds: `<title>` often adds the site name ("What's new — pandas
131+
docs") to the page's own heading ("What's new").
132+
"""
133+
first = next((line for line in content.splitlines() if line.strip()), "")
134+
line, wanted = _normalized(_visible(first)), _normalized(title)
135+
if not line or not wanted:
136+
return False
137+
if wanted in line:
138+
return True
139+
return first.lstrip().startswith("#") and line in wanted
140+
141+
142+
def with_page_title(html: str, content: str) -> str:
143+
"""``content`` with the page's title as a heading on top, if it lacks one."""
144+
if content.startswith(_FAILURE_PREFIX):
145+
return content
146+
title = page_title(html)
147+
if title is None or _opens_with_title(content, title):
148+
return content
149+
heading = _MARKDOWN_SPECIAL.sub(r"\\\1", title)
150+
return f"# {heading}\n\n{content.lstrip()}"

0 commit comments

Comments
 (0)