LeoVasanko
31b56c3e5e
/pages.md (front page /.md) serves the page's source Markdown — per-language hybrid assembly, title injected as '# title' when the content has no h1 — with the same ETag/Last-Modified/no-cache scheme as the HTML, which now carries vary: accept. Pages advertise the counterpart via <link rel=alternate type=text/markdown> and llms.txt links point at the .md URLs.
304 lines
12 KiB
Python
304 lines
12 KiB
Python
"""Public content pages: front page, sitemap, robots, and the catch-all.
|
|
|
|
``GET /{path:path}`` resolves a slug path against the menu tree and renders
|
|
the page (or a category placeholder, or 404); it must be registered AFTER
|
|
the fastapi-vue asset routes so built frontend files win over content slugs
|
|
(see app.py). Every served document is recorded raw in analytics (one
|
|
access-log line with its true HTTP status; classification happens at
|
|
display time — see pagerite/analytics.py), as are robots.txt and sitemap.xml
|
|
fetches (they surface as crawler hits, since no activity message ever
|
|
follows them).
|
|
"""
|
|
|
|
import logging
|
|
from datetime import UTC, datetime
|
|
from email.utils import format_datetime
|
|
from xml.sax.saxutils import escape as xml_escape
|
|
|
|
from fastapi import APIRouter, HTTPException, Request
|
|
from fastapi.responses import RedirectResponse, Response
|
|
|
|
from pagerite import i18n, state
|
|
from pagerite.chunks import chunk_key
|
|
from pagerite.data import Node, node_markdown, resolve, sorted_nodes
|
|
from pagerite.markdown import has_h1
|
|
from pagerite.state import (
|
|
SITE_URL,
|
|
_html_response,
|
|
_is_reserved,
|
|
data,
|
|
)
|
|
from pagerite.tracking import _record_get
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
def _http_date(dt: datetime) -> str:
|
|
"""RFC 7231 date for the Last-Modified header."""
|
|
return format_datetime(dt.astimezone(UTC), usegmt=True)
|
|
|
|
|
|
def _is_trackable_path(path: str) -> bool:
|
|
"""Content URLs only: skip auth endpoints and reserved/machinery paths."""
|
|
if not path:
|
|
return True
|
|
if path == "auth" or path.startswith("auth/"):
|
|
return False
|
|
return not _is_reserved(path)
|
|
|
|
|
|
def _markdown_response(
|
|
request: Request, path: str, node: Node, lang: str, original: str
|
|
) -> Response:
|
|
"""The page as Markdown (text/markdown): the same source the HTML page
|
|
renders (per-language hybrid assembly), with the page title injected as
|
|
``# title`` when the content has no h1 of its own — mirroring
|
|
markdown.render's title injection, so the .md reads like the page.
|
|
|
|
Served at the page's ``.md`` URL, and also at the plain URL when the
|
|
request's Accept header explicitly names text/markdown (Vary: accept).
|
|
Language selection is the page's own (?lang= or Accept-Language), so
|
|
``/page.md?lang=fi`` is the Finnish markdown."""
|
|
content = node_markdown(data, node) or ""
|
|
title = node.title
|
|
if lang != original and lang in node.langs:
|
|
content = i18n.hybrid_markdown(data, node, path, lang)
|
|
if t := data.trans.get(chunk_key(node.title), {}).get(lang):
|
|
title = t
|
|
if title and not has_h1(content):
|
|
content = f"# {title}\n\n{content}"
|
|
etag = f'"{path}@{node.modified.timestamp()}g{state._render_gen}l{lang}md"'
|
|
headers = {
|
|
"etag": etag,
|
|
"last-modified": _http_date(node.modified),
|
|
"cache-control": "no-cache",
|
|
"vary": "accept",
|
|
}
|
|
if request.headers.get("if-none-match") == etag:
|
|
return Response(status_code=304, headers=headers)
|
|
if _is_trackable_path(path):
|
|
_record_get(request, lang=lang)
|
|
return Response(content, media_type="text/markdown", headers=headers)
|
|
|
|
|
|
@router.get("/")
|
|
async def front_page(request: Request) -> Response:
|
|
"""Render the front page (slug path "")."""
|
|
return await show_page(request, "")
|
|
|
|
|
|
@router.get("/sitemap.xml")
|
|
async def sitemap(request: Request) -> Response:
|
|
"""Dynamically generate a sitemap of all published article pages, plus
|
|
the machine-readable exports (feeds, llms.txt)."""
|
|
base = SITE_URL or str(request.base_url).rstrip("/")
|
|
entries: list[tuple[str, datetime, int]] = []
|
|
|
|
def walk(
|
|
nodes: dict[str, Node], prefix: str, parent_has_content: bool = True
|
|
) -> None:
|
|
first_content_slug = next(
|
|
(
|
|
slug
|
|
for slug, node in sorted_nodes(nodes)
|
|
if node.published and node.chunks is not None
|
|
),
|
|
None,
|
|
)
|
|
for slug, node in sorted_nodes(nodes):
|
|
path = f"{prefix}/{slug}" if prefix else slug
|
|
depth = path.count("/") if path else 0
|
|
if (
|
|
not parent_has_content
|
|
and slug == first_content_slug
|
|
and node.published
|
|
and node.chunks is not None
|
|
and depth > 0
|
|
):
|
|
depth -= 1
|
|
if node.published and node.chunks is not None:
|
|
entries.append((path, node.modified, depth))
|
|
if node.children:
|
|
walk(node.children, path, node.chunks is not None)
|
|
|
|
walk(data.menu, "")
|
|
|
|
def priority(depth: int) -> float:
|
|
return max(0.1, 1.0 - depth * 0.2)
|
|
|
|
lines = [
|
|
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
|
|
]
|
|
for path, modified, depth in entries:
|
|
loc = xml_escape(f"{base}/{path}" if path else base)
|
|
lastmod = (
|
|
modified.astimezone(UTC)
|
|
.replace(microsecond=0)
|
|
.isoformat()
|
|
.replace("+00:00", "Z")
|
|
)
|
|
lines.append(
|
|
f" <url>"
|
|
f"<loc>{loc}</loc>"
|
|
f"<lastmod>{lastmod}</lastmod>"
|
|
f"<priority>{priority(depth):.1f}</priority>"
|
|
f"</url>"
|
|
)
|
|
lines.append("</urlset>")
|
|
|
|
# The machine-readable exports (feeds, llms.txt) are linked too, with
|
|
# the latest article's modification time as their lastmod.
|
|
if entries:
|
|
latest = max(m for _, m, _ in entries)
|
|
lastmod = (
|
|
latest.astimezone(UTC)
|
|
.replace(microsecond=0)
|
|
.isoformat()
|
|
.replace("+00:00", "Z")
|
|
)
|
|
for special in ("llms.txt", "feed.json", "feed.xml"):
|
|
lines.insert(
|
|
-1,
|
|
f" <url><loc>{xml_escape(f'{base}/{special}')}</loc>"
|
|
f"<lastmod>{lastmod}</lastmod></url>",
|
|
)
|
|
|
|
# Recorded like a page GET: never followed by an activity message, so
|
|
# it lands in the crawler list at display time (docs/analytics.md).
|
|
_record_get(request)
|
|
return Response(
|
|
"\n".join(lines),
|
|
media_type="application/xml",
|
|
headers={"cache-control": "no-cache"},
|
|
)
|
|
|
|
|
|
@router.get("/robots.txt")
|
|
async def robots_txt(request: Request) -> Response:
|
|
"""Allow content crawling, keep the SSO login (/auth/) and the
|
|
admin-gated API (/_api) out of search results, and point crawlers at
|
|
the sitemap."""
|
|
base = SITE_URL or str(request.base_url).rstrip("/")
|
|
body = f"User-agent: *\nAllow: /\nDisallow: /auth/\nDisallow: /_api\nSitemap: {base}/sitemap.xml\n"
|
|
_record_get(request)
|
|
return Response(
|
|
body,
|
|
media_type="text/plain",
|
|
headers={"cache-control": "no-cache"},
|
|
)
|
|
|
|
|
|
@router.get("/{path:path}", response_model=None)
|
|
async def show_page(request: Request, path: str) -> Response:
|
|
"""Render the content page at a slug path, or 404.
|
|
|
|
A node without content is a category label: its URL renders a
|
|
placeholder page (nav links point straight at its first child).
|
|
"""
|
|
path = path.strip("/")
|
|
accept_language = request.headers.get("accept-language", "")
|
|
# A .md suffix (or an Accept header explicitly naming text/markdown)
|
|
# asks for the page's Markdown instead of HTML. Slugs never contain a
|
|
# dot, so "page.md" unambiguously means the markdown of "page"; the
|
|
# front page's markdown URL is "/.md".
|
|
markdown = False
|
|
if path.endswith(".md"):
|
|
path = path[:-3]
|
|
markdown = True
|
|
if path and _is_reserved(path):
|
|
# Invalid slug shape: not a content URL, let FastAPI return its
|
|
# built-in 404 instead of rendering an editable article page.
|
|
# Recorded like any other GET: telltale scanner paths (dotpaths
|
|
# like /.env, *.php) classify the IP as abuse at display time.
|
|
_record_get(request, status=404)
|
|
raise HTTPException(404)
|
|
chain = resolve(data.menu, path)
|
|
node = chain[-1] if chain else None
|
|
if node is not None and node.published and node.chunks is not None:
|
|
# Language selection (docs/localization.md): ?lang= wins when a
|
|
# translation exists, else header logic. Analytics keep the raw
|
|
# Accept-Language header regardless of the selection, and record
|
|
# the resolved language as the GET's rendered language.
|
|
query_lang = request.query_params.get("lang")
|
|
original = i18n.primary_lang(data.menu, path)
|
|
lang = i18n.select_language(
|
|
query_lang,
|
|
accept_language,
|
|
lambda tag: tag in node.langs,
|
|
original=original,
|
|
)
|
|
# A ?lang= override is replicated onto the page's navigation links
|
|
# (link_lang), so clicks and prefetches stay in the chosen language.
|
|
# Query and header-selected renders of the same language differ in
|
|
# their links, so link_lang is part of the ETag and body cache key.
|
|
link_lang = i18n.base_tag(query_lang or "")
|
|
if markdown or "text/markdown" in request.headers.get("accept", ""):
|
|
return _markdown_response(request, path, node, lang, original)
|
|
# (browsers would otherwise cache heuristically and serve stale
|
|
# pages, e.g. after a theme change). In-session speed instead comes
|
|
# from pagerite.js's in-memory page cache (preload everything, never
|
|
# fetch on navigation); the ETag just makes those one-time preload
|
|
# fetches and any revalidation cheap.
|
|
etag = f'"{path}@{node.modified.timestamp()}g{state._render_gen}l{lang}q{link_lang}"'
|
|
if request.headers.get("if-none-match") == etag:
|
|
return Response(status_code=304)
|
|
if _is_trackable_path(path):
|
|
_record_get(request, lang=lang)
|
|
return _html_response(
|
|
request,
|
|
"page",
|
|
path,
|
|
headers={
|
|
"etag": etag,
|
|
"last-modified": _http_date(node.modified),
|
|
"cache-control": "no-cache",
|
|
# The same URL serves text/markdown when the Accept header
|
|
# explicitly asks for it (_markdown_response).
|
|
"vary": "accept",
|
|
},
|
|
lang=lang,
|
|
link_lang=link_lang,
|
|
)
|
|
if node is not None and node.published and node.chunks is None:
|
|
# Category label without a landing page: placeholder with the pen
|
|
# to create it (404 — no page here, but the node is real).
|
|
# Language selection as on content pages, but over the whole
|
|
# subtree's availability: the category has no chunks of its own —
|
|
# its heading, the navigation and the cards' text localize from
|
|
# the title map and the target articles' translations.
|
|
query_lang = request.query_params.get("lang")
|
|
subtree_langs = i18n.subtree_languages(node)
|
|
lang = i18n.select_language(
|
|
query_lang,
|
|
accept_language,
|
|
lambda tag: tag in subtree_langs,
|
|
original=i18n.primary_lang(data.menu, path),
|
|
)
|
|
link_lang = i18n.base_tag(query_lang or "")
|
|
if _is_trackable_path(path):
|
|
_record_get(request, status=404, lang=lang)
|
|
return _html_response(
|
|
request,
|
|
"category",
|
|
path,
|
|
404,
|
|
headers={
|
|
"last-modified": _http_date(node.modified),
|
|
"cache-control": "no-cache",
|
|
},
|
|
lang=lang,
|
|
link_lang=link_lang,
|
|
)
|
|
if node is None and not path:
|
|
# No front page (no top-level node with slug ""): "/" opens the
|
|
# first item of the navigation instead.
|
|
for slug, item in sorted_nodes(data.menu):
|
|
if item.published:
|
|
return RedirectResponse(f"/{slug}")
|
|
if _is_trackable_path(path):
|
|
_record_get(request, status=404)
|
|
return _html_response(request, "not-found", path, 404)
|