From 31b56c3e5ee4fb9ce31c0b2f378974cb07c64308 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Fri, 25 Sep 2026 18:04:47 +0000 Subject: [PATCH] Serve each article's Markdown at .md URLs (and via Accept: text/markdown) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /pages.md (front page /.md) serves the page's source Markdown — per-language hybrid assembly, title injected as '# title' when the content has no h1 — with the same ETag/Last-Modified/no-cache scheme as the HTML, which now carries vary: accept. Pages advertise the counterpart via and llms.txt links point at the .md URLs. --- AGENTS.md | 4 ++-- docs/backend.md | 4 ++-- pagerite/feeds.py | 5 +++-- pagerite/pages.py | 55 ++++++++++++++++++++++++++++++++++++++++++++--- pagerite/views.py | 6 ++++++ 5 files changed, 65 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index cf26734..937af8d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,8 +15,8 @@ Pagerite is a CMS. See `docs` for the full design and implementation details. Ke - `files.py` — `FileStore` (content-addressed, RAM-cached), image derivative helpers (`store_image`), file routes (`/_api/files`, `/_f/`, `/_themes/`, `/_fonts/`, favicon settings). - `api.py` — editor REST + WS: `/_api/pages`, `/_api/structure`, `/_api/settings`, `/_api/toggle-task`, `/_api/translations`, `/_api/ws/editor`, `/_translate/{key}`. - `tracking.py` — visit analytics: GeoIP, client enrichment, favicon fetch, `/_ws`, `/_api/ws/analytics`, the `/_a` page (docs/analytics.md). - - `pages.py` — public content pages: `/`, `/sitemap.xml`, `/robots.txt`, the `/{path:path}` catch-all. - - `feeds.py` — machine-readable exports: `/llms.txt`, `/feed.json` (JSON Feed 1.1), `/feed.xml` (RSS 2.0 + atom:link); all published articles, full content, linked from every page `` and the sitemap, recorded in analytics. + - `pages.py` — public content pages: `/`, `/sitemap.xml`, `/robots.txt`, the `/{path:path}` catch-all. Articles serve their Markdown (title injected as `# title`, per-language hybrid) at a `.md`-suffixed URL, or at the plain URL when Accept explicitly names text/markdown; pages link it via ``, and llms.txt links point at the `.md` URLs. + - `feeds.py` — machine-readable exports: `/llms.txt` (compact index linking each article's `.md` Markdown URL), `/feed.json` (JSON Feed 1.1), `/feed.xml` (RSS 2.0 + atom:link); the feeds carry all published articles with full content, linked from every page `` and the sitemap, recorded in analytics. - `data.py` — msgspec Structs for the kanta database. - `chunks.py` — block-level Markdown chunking and content-hash keys for the chunk stores (docs/migrate.md). - `i18n.py` — language selection, translation assembly (chunks + overrides) and translated-edit recording (per-chunk user overrides in `Data.overrides`, per-language title overrides, refresh). diff --git a/docs/backend.md b/docs/backend.md index 4804fdf..488374c 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -10,8 +10,8 @@ Thin FastAPI assembly: lifespan (open the kanta database, load the file store, t - `files.py` — the `FileStore` and image derivative helpers, and the file routes: `/_api/files`, `/_f/`, `/_themes/`, `/_fonts/`, the favicon settings endpoints. - `api.py` — the editor REST API and WebSockets: `/_api/pages`, `/_api/structure`, `/_api/settings`, `/_api/toggle-task`, `/_api/translations`, `/_api/ws/editor`, and the translator channel `/_translate/{clientkey}`. - `tracking.py` — visit analytics: GeoIP, client enrichment, favicon fetching, debounced broadcasts, the `/_ws` activity socket, the admin stream `/_api/ws/analytics`, and the `/_a` viewer page. -- `feeds.py` — machine-readable site exports: `/llms.txt` (Markdown site map for LLM agents), `/feed.json` (JSON Feed 1.1) and `/feed.xml` (RSS 2.0 + atom:link), all carrying every published article with full content (relative URLs absolutized), linked from every page's `` and from the sitemap, and recorded in analytics like page GETs. Bodies are RAM-cached (keyed by base URL, cleared by `_invalidate_pages` like the page render cache) and served with a content-hash ETag (304 revalidation). -- `pages.py` — the public content pages: `/`, `/sitemap.xml`, `/robots.txt` and the `/{path:path}` catch-all. +- `feeds.py` — machine-readable site exports: `/llms.txt` (Markdown site map for LLM agents, linking each article's `.md` Markdown URL), `/feed.json` (JSON Feed 1.1) and `/feed.xml` (RSS 2.0 + atom:link), the feeds carrying every published article with full content (relative URLs absolutized), linked from every page's `` and from the sitemap, and recorded in analytics like page GETs. Bodies are RAM-cached (keyed by base URL, cleared by `_invalidate_pages` like the page render cache) and served with a content-hash ETag (304 revalidation). +- `pages.py` — the public content pages: `/`, `/sitemap.xml`, `/robots.txt` and the `/{path:path}` catch-all. Every article also has a Markdown counterpart: a `.md` suffix on the page URL (or an Accept header explicitly naming `text/markdown` on the plain URL — the HTML response carries `vary: accept`) serves the page's source Markdown (per-language hybrid assembly via `?lang=`/Accept-Language, page title injected as `# title` when the content has no h1, same ETag/Last-Modified scheme as the HTML). Pages advertise it with ``. Route ordering is load-bearing and lives in `app.py`: the api/tracking/files routers are included BEFORE `frontend.route(app, "/")` is called — fastapi-vue inserts its file routes at the position where `route()` was called (during `load()` in the lifespan), so anything registered earlier wins. The content catch-all `/{path:path}` is included AFTER `frontend.route()` so that built frontend assets still take priority over content slugs. The `Frontend` is constructed with `spa=False` explicitly: it only serves the built files without a catch-all. diff --git a/pagerite/feeds.py b/pagerite/feeds.py index 24a5692..7032457 100644 --- a/pagerite/feeds.py +++ b/pagerite/feeds.py @@ -2,7 +2,8 @@ - ``/llms.txt`` (llmstxt.org convention): a Markdown map of the site for LLM agents — the brand as title, then every published article as a link - with a short excerpt. + to its ``.md`` Markdown counterpart (pages.show_page) with a short + excerpt. - ``/feed.json``: JSON Feed 1.1 of all published articles, full content. - ``/feed.xml``: the same as RSS 2.0 (with an atom:link self reference) for older feed readers. @@ -78,7 +79,7 @@ def _iso(node: Node) -> str: def _render_llms(base: str) -> str: lines = [f"# {data.brand}", "", "## Pages", ""] for path, node in _articles(): - url = f"{base}/{path}" if path else base + url = f"{base}/{path}.md" if path else f"{base}/.md" excerpt = _description(_body_html(node, base), 120) suffix = f": {excerpt}" if excerpt else "" lines.append(f"- [{node.title or path}]({url}){suffix}") diff --git a/pagerite/pages.py b/pagerite/pages.py index 46e9b59..de08264 100644 --- a/pagerite/pages.py +++ b/pagerite/pages.py @@ -19,7 +19,9 @@ from fastapi import APIRouter, HTTPException, Request from fastapi.responses import RedirectResponse, Response from pagerite import i18n, state -from pagerite.data import Node, resolve, sorted_nodes +from pagerite.chunks import chunk_key +from pagerite.data import Node, node_markdown, resolve, sorted_nodes +from pagerite.markdown import has_h1 from pagerite.state import ( SITE_URL, _html_response, @@ -47,6 +49,40 @@ def _is_trackable_path(path: str) -> bool: return not _is_reserved(path) +def _markdown_response( + request: Request, path: str, node: Node, lang: str, original: str +) -> Response: + """The page as Markdown (text/markdown): the same source the HTML page + renders (per-language hybrid assembly), with the page title injected as + ``# title`` when the content has no h1 of its own — mirroring + markdown.render's title injection, so the .md reads like the page. + + Served at the page's ``.md`` URL, and also at the plain URL when the + request's Accept header explicitly names text/markdown (Vary: accept). + Language selection is the page's own (?lang= or Accept-Language), so + ``/page.md?lang=fi`` is the Finnish markdown.""" + content = node_markdown(data, node) or "" + title = node.title + if lang != original and lang in node.langs: + content = i18n.hybrid_markdown(data, node, path, lang) + if t := data.trans.get(chunk_key(node.title), {}).get(lang): + title = t + if title and not has_h1(content): + content = f"# {title}\n\n{content}" + etag = f'"{path}@{node.modified.timestamp()}g{state._render_gen}l{lang}md"' + headers = { + "etag": etag, + "last-modified": _http_date(node.modified), + "cache-control": "no-cache", + "vary": "accept", + } + if request.headers.get("if-none-match") == etag: + return Response(status_code=304, headers=headers) + if _is_trackable_path(path): + _record_get(request, lang=lang) + return Response(content, media_type="text/markdown", headers=headers) + + @router.get("/") async def front_page(request: Request) -> Response: """Render the front page (slug path "").""" @@ -164,6 +200,14 @@ async def show_page(request: Request, path: str) -> Response: """ path = path.strip("/") accept_language = request.headers.get("accept-language", "") + # A .md suffix (or an Accept header explicitly naming text/markdown) + # asks for the page's Markdown instead of HTML. Slugs never contain a + # dot, so "page.md" unambiguously means the markdown of "page"; the + # front page's markdown URL is "/.md". + markdown = False + if path.endswith(".md"): + path = path[:-3] + markdown = True if path and _is_reserved(path): # Invalid slug shape: not a content URL, let FastAPI return its # built-in 404 instead of rendering an editable article page. @@ -179,18 +223,20 @@ async def show_page(request: Request, path: str) -> Response: # Accept-Language header regardless of the selection, and record # the resolved language as the GET's rendered language. query_lang = request.query_params.get("lang") + original = i18n.primary_lang(data.menu, path) lang = i18n.select_language( query_lang, accept_language, lambda tag: tag in node.langs, - original=i18n.primary_lang(data.menu, path), + original=original, ) # A ?lang= override is replicated onto the page's navigation links # (link_lang), so clicks and prefetches stay in the chosen language. # Query and header-selected renders of the same language differ in # their links, so link_lang is part of the ETag and body cache key. link_lang = i18n.base_tag(query_lang or "") - # no-cache forbids serving a stored page without revalidation + if markdown or "text/markdown" in request.headers.get("accept", ""): + return _markdown_response(request, path, node, lang, original) # (browsers would otherwise cache heuristically and serve stale # pages, e.g. after a theme change). In-session speed instead comes # from pagerite.js's in-memory page cache (preload everything, never @@ -209,6 +255,9 @@ async def show_page(request: Request, path: str) -> Response: "etag": etag, "last-modified": _http_date(node.modified), "cache-control": "no-cache", + # The same URL serves text/markdown when the Accept header + # explicitly asks for it (_markdown_response). + "vary": "accept", }, lang=lang, link_lang=link_lang, diff --git a/pagerite/views.py b/pagerite/views.py index ceaf4bd..96dbe41 100644 --- a/pagerite/views.py +++ b/pagerite/views.py @@ -336,6 +336,7 @@ def _layout( lang: str = i18n.ORIGINAL_LANGUAGE, canonical: str = "", alternates: list[tuple[str, str]] = (), + markdown_url: str = "", ) -> Template: """Page layout template with standard assets and ES-module scripts. @@ -391,6 +392,9 @@ def _layout( rel="alternate", type="application/rss+xml", title="RSS feed", href="/feed.xml" ) doc.link(rel="llms-txt", type="text/markdown", title="llms.txt", href="/llms.txt") + # The page's own Markdown counterpart (pages.show_page's .md mode). + if markdown_url: + doc.link(rel="alternate", type="text/markdown", href=markdown_url) for key, value in (social or {}).items(): if value: if key.startswith(("og:", "article:")): @@ -1397,6 +1401,8 @@ def render_page( lang, canonical, alternates, + # The page's Markdown counterpart (pages.show_page's .md mode). + f"/{path}.md", )( Title=f"{title} – {brand}" if brand else title, Brand=_brand_link(brand, brand_html, link_lang),