Repair the rest of what Google's export does to frontmatter

backend/integrations/google/indexer.py

@@ -19,6 +19,7 @@
 
 from __future__ import annotations
 
+import json
 import logging
 import re
 from datetime import datetime
@@ -27,6 +28,7 @@
 import httpx
 
 from ...services import source_service
+from ...services.skill_service import FRONTMATTER_SCAN_BYTES, declared_skill
 from ..storage import get_valid_token
 
 logger = logging.getLogger(__name__)
@@ -101,21 +103,97 @@ async def _target_modified_time(client: httpx.AsyncClient, file_id: str) -> str
 
 
 _ESCAPED_RULE_RE = re.compile(r"^\\(-{3,})[ \t]*$", re.MULTILINE)
+# Still-escaped delimiters are accepted here because a leading BOM blocks the
+# MULTILINE ^ above from ever seeing the first line.
+_DELIMITER_RE = re.compile(r"\\?-{3,}[ \t]*")
+# A bare * or _ in the export is the exporter's styling markup (bold, italic):
+# the author's own characters always arrive backslash-escaped, which the
+# lookbehind preserves for the unescape pass to restore.
+_EMPHASIS_RE = re.compile(r"(?<!\\)[*_]+")
+# CommonMark's full escapable-punctuation set — the exporter may escape any of
+# these, and one surviving backslash inside a quoted value breaks json.loads.
+_MARKDOWN_ESCAPE_RE = re.compile(r"\\([!\"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~])")
+# Docs curls typed quotes by default; both pairs mean "the author quoted this".
+_QUOTE_PAIRS = (("\u201c", "\u201d"), ("\u2018", "\u2019"), ('"', '"'))
+
+
+def repair_exported_frontmatter(markdown: str) -> str:
+    r"""Undo what Google's markdown export does to a frontmatter block.
+
+    A skill authored in a Doc is written as prose and exported as markdown, and
+    the exporter marks up characters the author never typed: `---` becomes
+    `\---`, an underscore becomes `\_`, a styled key becomes `**name:**`, a
+    typed quote curls. None of it is visible in the editor and all of it is
+    fatal — the delimiters stop delimiting, or the value stops parsing, and the
+    document silently isn't a skill. What the author saw is what gets stored.
+
+    Escaped rules are repaired everywhere: a line of dashes was dashes when it
+    was written, wherever in the document it sits. Everything else is repaired
+    only between the delimiters, and the rewrite is kept ONLY when the result
+    declares a valid skill — an ordinary document that merely opens with a
+    horizontal rule is returned untouched, because prose between two rules is
+    the author's text, not frontmatter. Only the first FRONTMATTER_SCAN_BYTES
+    are considered, the same bound every skill surface reads.
+    """
+    text = _ESCAPED_RULE_RE.sub(r"\1", markdown)
+    head, tail = text[:FRONTMATTER_SCAN_BYTES], text[FRONTMATTER_SCAN_BYTES:]
+    repaired_head = _repaired_frontmatter_head(head)
+    if repaired_head is None:
+        return text
+    candidate = repaired_head + tail
+    if declared_skill(candidate[:FRONTMATTER_SCAN_BYTES]) is None:
+        return text
+    return candidate
 
 
-def unescape_exported_rules(markdown: str) -> str:
-    r"""Undo the escaping Google's markdown export applies to a line of dashes.
+def _repaired_frontmatter_head(head: str) -> str | None:
+    """The head with its frontmatter block repaired, or None if it has none.
 
-    A Doc whose author typed `---` exports as `\---`, because bare dashes would
-    otherwise be a horizontal rule. That escaping is invisible to the author and
-    fatal to a frontmatter block: the delimiters stop being delimiters, so a
-    skill written in a Doc could never declare itself. The line was dashes when
-    it was written, so it is dashes again here.
+    A leading BOM or empty first paragraph — one invisible keystroke in a Doc —
+    is dropped so the block sits at the top, where every parser and the
+    `content LIKE '---%'` prefilter expect it. Delimiter lines are normalized to
+    exactly `---` so the repaired span and the span `parse_frontmatter` reads
+    are the same span.
+    """
+    lines = head.lstrip("\ufeff").split("\n")
+    start = 0
+    while start < len(lines) and not lines[start].strip():
+        start += 1
+    if start >= len(lines) or not _DELIMITER_RE.fullmatch(lines[start]):
+        return None
+    closing = next(
+        (i for i in range(start + 1, len(lines)) if _DELIMITER_RE.fullmatch(lines[i])), None
+    )
+    if closing is None:
+        return None
+    repaired = [_repair_frontmatter_line(line) for line in lines[start + 1 : closing]]
+    return "\n".join(["---", *repaired, "---", *lines[closing + 1 :]])
+
+
+def _repair_frontmatter_line(line: str) -> str:
+    r"""One `key: value` line, as the author wrote it in the Doc.
 
-    Trailing spaces are left alone — Google marks every line with them as a hard
-    break, and the frontmatter parser already ignores them.
+    Emphasis markers go because frontmatter has no formatting — a key the
+    author styled is still just the key. Backslashes go because the escaping is
+    the exporter's. The order is load-bearing: bare `*`/`_` are markup only
+    because the author's own are still escaped when emphasis is stripped.
     """
-    return _ESCAPED_RULE_RE.sub(r"\1", markdown)
+    line = _MARKDOWN_ESCAPE_RE.sub(r"\1", _EMPHASIS_RE.sub("", line))
+    key, colon, value = line.partition(":")
+    if not colon:
+        return line
+    return f"{key.strip()}: {_requoted_value(value.strip())}"
+
+
+def _requoted_value(value: str) -> str:
+    """A value the author wrapped in quotes, re-encoded as the JSON string the
+    frontmatter parser expects. json.dumps, not hand-built quoting: the interior
+    is the author's literal text, and it may contain quotes or backslashes of
+    its own. A curly quote inside the sentence is their text and stays curly."""
+    for opening, closing in _QUOTE_PAIRS:
+        if len(value) >= 2 and value.startswith(opening) and value.endswith(closing):
+            return json.dumps(value[1:-1], ensure_ascii=False)
+    return value
 
 
 async def _require_readable_folder(client: httpx.AsyncClient, folder_id: str) -> None:
@@ -493,7 +571,7 @@ async def extract_drive_text(
         from ...services.file_extraction import extract_text, is_pdf
 
         if mime == MIME_GOOGLE_DOC:
-            text = unescape_exported_rules(await _export(client, file_id, "text/markdown"))
+            text = repair_exported_frontmatter(await _export(client, file_id, "text/markdown"))
         elif mime == MIME_GOOGLE_SHEET:
             # XLSX export keeps every visible sheet (Drive's CSV export drops
             # everything except the first).

backend/migrations/versions/0186_repair_drive_skill_frontmatter.py

@@ -0,0 +1,160 @@
+"""Repair Google-export damage in already-extracted Drive documents.
+
+The extraction-time repair (repair_exported_frontmatter) only runs when a
+document is re-extracted, and extraction is keyed on Drive's own modifiedTime —
+so a document damaged before the repair shipped stays damaged until its author
+happens to edit it. This carries the stored rows forward once.
+
+The logic is a frozen copy of the extraction-time repair, per this directory's
+convention: a migration must keep producing the same result even after the live
+code moves on.
+
+A row is rewritten only when the repair turns it into a valid skill declaration
+it wasn't before, or when it already declared a skill whose values are wrapped
+in the curly quotes only Docs' exporter produces. A document that already
+declares a clean skill — e.g. a hand-authored SKILL.md synced from a Drive
+folder — stays exactly as its author wrote it.
+
+Revision ID: 0186
+Revises: 0185
+"""
+
+import json
+import re
+
+from alembic import op
+from sqlalchemy import text
+
+revision = "0186"
+down_revision = "0185"
+branch_labels = None
+depends_on = None
+
+FRONTMATTER_SCAN_BYTES = 8192
+MAX_SKILL_NAME_LENGTH = 64
+MAX_SKILL_DESCRIPTION_LENGTH = 1024
+
+_ESCAPED_RULE_RE = re.compile(r"^\\(-{3,})[ \t]*$", re.MULTILINE)
+_DELIMITER_RE = re.compile(r"\\?-{3,}[ \t]*")
+_EMPHASIS_RE = re.compile(r"(?<!\\)[*_]+")
+_MARKDOWN_ESCAPE_RE = re.compile(r"\\([!\"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~])")
+_QUOTE_PAIRS = (("\u201c", "\u201d"), ("\u2018", "\u2019"), ('"', '"'))
+_CURLY_PAIRS = (("\u201c", "\u201d"), ("\u2018", "\u2019"))
+
+
+def _repair(markdown: str) -> str:
+    text_ = _ESCAPED_RULE_RE.sub(r"\1", markdown)
+    head, tail = text_[:FRONTMATTER_SCAN_BYTES], text_[FRONTMATTER_SCAN_BYTES:]
+    repaired_head = _repaired_head(head)
+    if repaired_head is None:
+        return text_
+    candidate = repaired_head + tail
+    if not _declares_skill(candidate[:FRONTMATTER_SCAN_BYTES]):
+        return text_
+    return candidate
+
+
+def _repaired_head(head: str) -> str | None:
+    lines = head.lstrip("\ufeff").split("\n")
+    start = 0
+    while start < len(lines) and not lines[start].strip():
+        start += 1
+    if start >= len(lines) or not _DELIMITER_RE.fullmatch(lines[start]):
+        return None
+    closing = next(
+        (i for i in range(start + 1, len(lines)) if _DELIMITER_RE.fullmatch(lines[i])), None
+    )
+    if closing is None:
+        return None
+    repaired = [_repair_line(line) for line in lines[start + 1 : closing]]
+    return "\n".join(["---", *repaired, "---", *lines[closing + 1 :]])
+
+
+def _repair_line(line: str) -> str:
+    line = _MARKDOWN_ESCAPE_RE.sub(r"\1", _EMPHASIS_RE.sub("", line))
+    key, colon, value = line.partition(":")
+    if not colon:
+        return line
+    value = value.strip()
+    for opening, closing in _QUOTE_PAIRS:
+        if len(value) >= 2 and value.startswith(opening) and value.endswith(closing):
+            value = json.dumps(value[1:-1], ensure_ascii=False)
+            break
+    return f"{key.strip()}: {value}"
+
+
+def _frontmatter(md: str) -> dict | None:
+    if not md.startswith("---"):
+        return None
+    end = md.find("\n---", 3)
+    if end == -1:
+        return None
+    meta: dict = {}
+    for line in md[3:end].strip("\n").splitlines():
+        line = line.rstrip()
+        if not line or line.startswith("#") or ":" not in line:
+            continue
+        key, _, val = line.partition(":")
+        val = val.strip()
+        if val.startswith('"') and val.endswith('"'):
+            try:
+                val = str(json.loads(val))
+            except ValueError:
+                return None
+        meta[key.strip()] = val
+    return meta
+
+
+def _declares_skill(md: str) -> bool:
+    meta = _frontmatter(md)
+    if meta is None:
+        return False
+    name = meta.get("name", "").strip()
+    description = meta.get("description", "").strip()
+    return (
+        0 < len(name) <= MAX_SKILL_NAME_LENGTH
+        and 0 < len(description) <= MAX_SKILL_DESCRIPTION_LENGTH
+    )
+
+
+def _curly_wrapped(meta: dict) -> bool:
+    return any(
+        len(value) >= 2 and value.startswith(opening) and value.endswith(closing)
+        for value in meta.values()
+        if isinstance(value, str)
+        for opening, closing in _CURLY_PAIRS
+    )
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    rows = bind.execute(
+        text(
+            "SELECT id, content FROM drive_documents "
+            "WHERE deleted_at IS NULL AND content IS NOT NULL "
+            "  AND left(content, 64) LIKE '%---%'"
+        )
+    ).mappings()
+    for row in rows:
+        content = row["content"]
+        repaired = _repair(content)
+        if repaired == content:
+            continue
+        if not _declares_skill(repaired[:FRONTMATTER_SCAN_BYTES]):
+            continue
+        current = _frontmatter(content[:FRONTMATTER_SCAN_BYTES])
+        already_a_skill = _declares_skill(content[:FRONTMATTER_SCAN_BYTES])
+        if already_a_skill and not (current and _curly_wrapped(current)):
+            continue
+        bind.execute(
+            text(
+                "UPDATE drive_documents SET content = :content, "
+                "content_hash = md5(:content), embed_stale = TRUE, updated_at = now() "
+                "WHERE id = :row_id"
+            ),
+            {"content": repaired, "row_id": row["id"]},
+        )
+
+
+def downgrade() -> None:
+    pass