feat(seo): add meta description, canonical link and sitemap

app/main.py

@@ -6,10 +6,17 @@
 import time
 from contextlib import asynccontextmanager
 from pathlib import Path
+from xml.sax.saxutils import escape
 
 from fastapi import FastAPI, Form, HTTPException, Request
 from fastapi import Path as PathParam
-from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, RedirectResponse
+from fastapi.responses import (
+    HTMLResponse,
+    JSONResponse,
+    PlainTextResponse,
+    RedirectResponse,
+    Response,
+)
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 from slowapi import _rate_limit_exceeded_handler
@@ -275,8 +282,26 @@ async def _browser_icon_redirect():
 
 
 @app.get("/robots.txt", include_in_schema=False)
-async def robots_txt():
-    return PlainTextResponse(_ROBOTS_TXT)
+async def robots_txt(request: Request):
+    # Sitemap must be an absolute URL (robots.txt spec) and is emitted per
+    # request so self-hosters on any domain get a correct one without config.
+    return PlainTextResponse(f"{_ROBOTS_TXT}\nSitemap: {_abs_url(request, '/sitemap.xml')}\n")
+
+
+@app.get("/sitemap.xml", include_in_schema=False)
+async def sitemap_xml(request: Request):
+    # The landing page is the only indexable URL this app serves. Paste pages
+    # are unguessable capability URLs (see the noindex in paste.html) and would
+    # be a privacy leak if listed, so the sitemap is deliberately a single
+    # entry rather than a crawl of storage.
+    url = escape(_abs_url(request, "/"))
+    body = (
+        '<?xml version="1.0" encoding="UTF-8"?>\n'
+        '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
+        f"  <url><loc>{url}</loc><changefreq>monthly</changefreq><priority>1.0</priority></url>\n"
+        "</urlset>\n"
+    )
+    return Response(content=body, media_type="application/xml")
 
 
 app.include_router(api_router)

templates/base.html

@@ -18,6 +18,12 @@
     })();
   </script>
   <title>{% block title %}Ghostbit — Encrypted paste, zero knowledge{% endblock %}</title>
+  {# Indexing directives live in their own block so paste and error pages can
+     opt out of the index without having to restate every og:/twitter: tag. #}
+  {% block seo %}
+  <meta name="description" content="Ghostbit is a self-hosted, end-to-end encrypted paste service. Text is encrypted in your browser before upload — the server only ever stores ciphertext.">
+  <link rel="canonical" href="{{ abs_url(request, request.url.path) }}">
+  {% endblock %}
   {% block meta %}
   <meta property="og:title" content="Ghostbit — Encrypted paste, zero knowledge">
   <meta property="og:description" content="Self-hosted, end-to-end encrypted paste service. Your data never touches the server in plaintext.">

templates/error.html

@@ -2,6 +2,12 @@
 
 {% block title %}{{ code }} — Ghostbit{% endblock %}
 
+{% block seo %}
+{# Error pages carry no canonical: they are served under whatever path failed,
+   including expired paste IDs, and none of that belongs in an index. #}
+<meta name="robots" content="noindex, nofollow">
+{% endblock %}
+
 {% block header_actions %}{% endblock %}
 
 {% block page %}

templates/paste.html

@@ -2,6 +2,16 @@
 
 {% block title %}{{ paste.id }} — Ghostbit{% endblock %}
 
+{% block seo %}
+{# A paste URL is a capability: whoever holds it can fetch the ciphertext, and
+   the decryption key lives in the fragment. Letting one into a search index
+   leaks the capability to everyone, so paste pages are noindex and carry no
+   canonical. Link-preview unfurls (block meta) still work — those bots read the
+   page directly rather than going through the index. #}
+<meta name="robots" content="noindex, nofollow">
+<meta name="description" content="An end-to-end encrypted Ghostbit paste. Decryption happens in your browser.">
+{% endblock %}
+
 {% block meta %}
 {# Paste pages deliberately omit og:image. An encrypted paste has nothing to
    preview, so link unfurls (iMessage, Slack, …) fall back to the compact card

templates/raw.html

@@ -3,6 +3,7 @@
 <head>
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <meta name="robots" content="noindex, nofollow">
   <title>{{ paste.id }} — raw</title>
   <style>
     * { margin: 0; padding: 0; box-sizing: border-box; }

tests/test_api.py

@@ -449,6 +449,44 @@ async def test_security_txt(client):
     assert "Contact:" in r.text
 
 
+@pytest.mark.anyio
+async def test_robots_txt_points_at_sitemap(client):
+    r = await client.get("/robots.txt")
+    assert r.status_code == 200
+    # The Sitemap directive must be an absolute URL per the robots.txt spec.
+    assert "Sitemap: http" in r.text
+    assert "/sitemap.xml" in r.text
+
+
+@pytest.mark.anyio
+async def test_sitemap_lists_only_the_landing_page(client):
+    r = await client.get("/sitemap.xml")
+    assert r.status_code == 200
+    assert r.headers["content-type"].startswith("application/xml")
+    # Exactly one entry: listing paste URLs would leak capability URLs.
+    assert r.text.count("<loc>") == 1
+
+
+@pytest.mark.anyio
+async def test_paste_page_is_noindex(client):
+    created = await client.post("/api/v1/pastes", json=_fake_paste())
+    assert created.status_code == 201
+    r = await client.get(f"/{created.json()['id']}")
+    assert r.status_code == 200
+    assert 'name="robots" content="noindex, nofollow"' in r.text
+    # A noindex page must not also advertise itself as canonical.
+    assert 'rel="canonical"' not in r.text
+
+
+@pytest.mark.anyio
+async def test_landing_page_has_description_and_canonical(client):
+    r = await client.get("/")
+    assert r.status_code == 200
+    assert 'name="description"' in r.text
+    assert 'rel="canonical"' in r.text
+    assert "noindex" not in r.text
+
+
 @pytest.mark.anyio
 async def test_id_collision_retry(client, monkeypatch):
     """When the random ID generator collides, create_paste should retry