fix(mcp): call the Mintlify search endpoint the documentation moved to (#12578)

mcp_server/changelog.d/mcp-docs-search-endpoint.fixed.md

@@ -0,0 +1 @@
+`prowler_docs_search` returns results again: it calls the search endpoint docs.prowler.com moved to, since the one it used no longer exists, and each result now names the page's title, the section it matched and a URL anchored at that section

mcp_server/prowler_mcp_server/prowler_documentation/search_engine.py

@@ -8,12 +8,21 @@ class SearchResult(BaseModel):
     """Search result model."""
 
     path: str = Field(description="Document path")
-    title: str = Field(description="Document title")
-    url: str = Field(description="Documentation URL")
-    highlights: list[str] = Field(
-        description="Highlighted content snippets showing query matches with <mark><b> tags",
+    title: str = Field(description="Title of the page the match is on")
+    section: str = Field(
+        description="Heading of the section the match is in", default=""
+    )
+    breadcrumbs: list[str] = Field(
+        description="Where the page sits in the documentation, from the top-level group down to the page itself",
         default_factory=list,
     )
+    url: str = Field(
+        description="Documentation URL, anchored at the matching section when it has an anchor"
+    )
+    excerpt: str = Field(
+        description="Text of the matching section, which is a part of the page and not the whole of it",
+        default="",
+    )
     score: float = Field(
         description="Relevance score for the search result", default=0.0
     )
@@ -24,12 +33,10 @@ class ProwlerDocsSearchEngine:
 
     def __init__(self):
         """Initialize the search engine."""
-        self.api_base_url = (
-            "https://api.mintlifytrieve.com/api/chunk_group/group_oriented_autocomplete"
-        )
-        self.dataset_id = "0096ba11-3f72-463b-9d95-b788495ac392"
-        self.api_key = "tr-T6JLeTkFXeNbNPyhijtI9XhIncydQQ3O"
-        self.docs_base_url = "https://prowler.mintlify.app"
+        # The endpoint docs.prowler.com itself calls, with the site's Mintlify
+        # project name as the last segment.
+        self.api_base_url = "https://leaves.mintlify.com/api/search/prowler"
+        self.docs_base_url = "https://docs.prowler.com"
 
         # HTTP client for Mintlify API
         self.mintlify_client = httpx.Client(
@@ -38,9 +45,6 @@ class ProwlerDocsSearchEngine:
                 "Content-Type": "application/json",
                 "Accept": "application/json",
                 "User-Agent": f"prowler-mcp-server/{__version__}",
-                "TR-Dataset": self.dataset_id,
-                "Authorization": self.api_key,
-                "X-API-Version": "V2",
             },
         )
 
@@ -59,75 +63,51 @@ class ProwlerDocsSearchEngine:
 
         Args:
             query: Search query string
-            page_size: Maximum number of results to return
+            page_size: Maximum number of results to return. The API decides how
+                many matches it answers with and takes no size of its own, so
+                this only trims the list it returned.
 
         Returns:
             list of search results
         """
         try:
-            # Construct request body
-            payload = {
-                "query": query,
-                "search_type": "fulltext",
-                "extend_results": True,
-                "highlight_options": {
-                    "highlight_window": 10,
-                    "highlight_max_num": 1,
-                    "highlight_max_length": 2,
-                    "highlight_strategy": "exactmatch",
-                    "highlight_delimiters": ["?", ",", ".", "!", "\n"],
-                },
-                "score_threshold": 0.2,
-                "filters": {"must_not": [{"field": "tag_set", "match": ["code"]}]},
-                "page_size": page_size,
-                "group_size": 3,
-            }
-
             # Make request to Mintlify API
             response = self.mintlify_client.post(
                 self.api_base_url,
-                json=payload,
+                json={"query": query, "filters": {}},
             )
             response.raise_for_status()
             data = response.json()
 
             # Parse results
             results = []
-            for result in data.get("results", []):
-                group = result.get("group", {})
-                chunks = result.get("chunks", [])
+            for match in data.get("results", [])[:page_size]:
+                metadata = match.get("metadata", {})
+                breadcrumbs = metadata.get("breadcrumbs", [])
+                doc_path = match.get("page", "")
 
-                # Get document path and title from group
-                doc_path = group.get("name", "")
-                group_title = group.get("name", "").replace("/", " / ").title()
+                # A match is one section of a page rather than the page: the
+                # heading it was found under is its header, and the page's own
+                # title is the last step of its breadcrumb trail.
+                section = match.get("header", "")
+                title = breadcrumbs[-1] if breadcrumbs else section
 
-                # If chunks exist, use the first chunk's title from metadata
-                title = group_title
-                if chunks:
-                    first_chunk = chunks[0].get("chunk", {})
-                    metadata = first_chunk.get("metadata", {})
-                    title = metadata.get("title", group_title)
-
-                # Construct full URL to docs
-                full_url = f"{self.docs_base_url}/{doc_path}"
-
-                # Extract highlights and scores from chunks
-                highlights = []
-                max_score = 0.0
-                for chunk_data in chunks:
-                    chunk_highlights = chunk_data.get("highlights", [])
-                    highlights.extend(chunk_highlights)
-                    # Track the highest score among all chunks in this group
-                    chunk_score = chunk_data.get("score", 0.0)
-                    max_score = max(max_score, chunk_score)
+                # Sent as "" for the section a page opens with and as null for
+                # the pages that have no anchors at all; both mean the page.
+                anchor = metadata.get("hash")
+                url = f"{self.docs_base_url}/{doc_path}"
+                if anchor:
+                    url = f"{url}#{anchor}"
 
                 results.append(
                     SearchResult(
                         path=doc_path,
                         title=title,
-                        url=full_url,
-                        highlights=highlights,
-                        score=max_score,
+                        section=section,
+                        breadcrumbs=breadcrumbs,
+                        url=url,
+                        excerpt=match.get("content", ""),
+                        score=match.get("score", 0.0),
                     )
                 )
 

mcp_server/prowler_mcp_server/prowler_documentation/server.py

@@ -17,9 +17,9 @@ def search(
     term: str = Field(description="The term to search for in the documentation"),
     page_size: int = Field(
         5,
-        description="Number of top results to return to return. It must be between 1 and 20.",
-        gt=1,
-        lt=20,
+        description="Number of top results to return. It must be between 1 and 20.",
+        ge=1,
+        le=20,
     ),
 ) -> list[dict[str, Any]]:
     """Search in Prowler documentation.
@@ -27,11 +27,12 @@ def search(
     This tool searches through the official Prowler documentation
     to find relevant information about everything related to Prowler.
 
-    Uses fulltext search to find the most relevant documentation pages
-    based on your query.
+    A result is one section of a documentation page, not the page itself: its
+    'excerpt' is that section alone. Read the whole page with
+    `prowler_docs_get_document`, passing the result's 'path'.
 
     Returns:
-        List of search results with highlights showing matched terms (in <mark><b> tags)
+        List of matching documentation sections, most relevant first
     """
     return prowler_docs_search_engine.search(term, page_size)  # type: ignore In the hint we cannot put SearchResult type because JSON API MCP Generator cannot handle Pydantic models yet
 

mcp_server/tests/prowler_documentation/test_server.py

@@ -0,0 +1,109 @@
+"""Tests for the Prowler documentation search tool.
+
+Mintlify moved the docs search to a new endpoint that answers with page
+sections, so a result is a part of a page and has to read as one.
+"""
+
+import json
+
+from fastmcp import Client
+
+SEARCH = "/api/search/prowler"
+
+
+def search_match(
+    path: str = "getting-started/installation",
+    *,
+    header: str = "Requirements",
+    breadcrumbs: tuple[str, ...] = ("Get Started", "Installation"),
+    anchor: str | None = "requirements",
+    score: float = 4.9,
+):
+    """One match as Mintlify answers with it: a section of a page, not the page."""
+    return {
+        "page": path,
+        "header": header,
+        "content": "Prowler runs on Python 3.9 or later.",
+        "metadata": {
+            "title": header,
+            "breadcrumbs": list(breadcrumbs),
+            "icon": "",
+            "hash": anchor,
+            "openapi": "",
+        },
+        "score": score,
+    }
+
+
+def stub_search_hit(docs_router, *matches):
+    """Serve the search endpoint, with one default match when none are given."""
+    if not matches:
+        matches = (search_match(),)
+    return docs_router.add("POST", SEARCH, json={"results": list(matches)})
+
+
+async def test_search_returns_the_matching_sections(mcp_root_server, docs_router):
+    """Every field of a result, since the shape of one changed with the endpoint."""
+    stub_search_hit(docs_router)
+
+    async with Client(mcp_root_server) as client:
+        result = await client.call_tool("prowler_docs_search", {"term": "install"})
+
+    match = result.data[0]
+    assert match["path"] == "getting-started/installation"
+    # The page's title, so a result reads as more than the heading it matched.
+    assert match["title"] == "Installation"
+    assert match["section"] == "Requirements"
+    assert match["breadcrumbs"] == ["Get Started", "Installation"]
+    assert match["excerpt"] == "Prowler runs on Python 3.9 or later."
+    assert match["score"] == 4.9
+    # Anchored: a match is a section, and the page it is on can be a long one.
+    assert match["url"] == (
+        "https://docs.prowler.com/getting-started/installation#requirements"
+    )
+
+
+async def test_the_search_query_is_sent_as_the_api_expects_it(
+    mcp_root_server, docs_router
+):
+    """The endpoint takes a POST body, not the payload the old one took."""
+    stub_search_hit(docs_router)
+
+    async with Client(mcp_root_server) as client:
+        await client.call_tool("prowler_docs_search", {"term": "install"})
+
+    request = docs_router.request_for("POST", SEARCH)
+    assert json.loads(request.content) == {"query": "install", "filters": {}}
+
+
+async def test_a_section_with_no_anchor_links_to_the_page(mcp_root_server, docs_router):
+    """The API sends "" for a page's first section and null for pages without anchors."""
+    stub_search_hit(
+        docs_router,
+        search_match(anchor=""),
+        search_match(path="getting-started/requirements", anchor=None),
+    )
+
+    async with Client(mcp_root_server) as client:
+        result = await client.call_tool("prowler_docs_search", {"term": "install"})
+
+    assert result.data[0]["url"] == (
+        "https://docs.prowler.com/getting-started/installation"
+    )
+    assert result.data[1]["url"] == (
+        "https://docs.prowler.com/getting-started/requirements"
+    )
+
+
+async def test_page_size_caps_a_response_the_api_did_not_size(
+    mcp_root_server, docs_router
+):
+    """The endpoint takes no size argument, so the cap has to be applied here."""
+    stub_search_hit(docs_router, *(search_match() for _ in range(6)))
+
+    async with Client(mcp_root_server) as client:
+        result = await client.call_tool(
+            "prowler_docs_search", {"term": "install", "page_size": 2}
+        )
+
+    assert len(result.data) == 2