feat(library): import curated collections + upload your own documents

helper-scripts/fetch-source-data.sh

@@ -16,7 +16,9 @@
 #     --dest DIR        library directory          (default: ./public/sources)
 #     --seed DIR        copy DIR/* into the library first (no clobber)
 #     --category CAT    only fetch one category    (e.g. 200_Medical)
+#     --collection ID   only fetch one collection  (e.g. austere-medicine)
 #     --list            print the manifest and exit
+#     --list-collections print the declared collections and exit
 #     --validate        lint the manifest (no network) and exit
 #     --force           re-download files that already exist
 #     --strict          exit non-zero if any download failed
@@ -31,6 +33,7 @@ MANIFEST="sources.yaml"
 DEST="public/sources"
 SEED=""
 ONLY_CATEGORY=""
+ONLY_COLLECTION=""
 MODE="fetch"
 FORCE=0
 STRICT=0
@@ -41,7 +44,9 @@ while [[ $# -gt 0 ]]; do
         --dest)     DEST="$2";     shift 2 ;;
         --seed)     SEED="$2";     shift 2 ;;
         --category) ONLY_CATEGORY="$2"; shift 2 ;;
+        --collection) ONLY_COLLECTION="$2"; shift 2 ;;
         --list)     MODE="list";     shift ;;
+        --list-collections) MODE="list-collections"; shift ;;
         --validate) MODE="validate"; shift ;;
         --force)    FORCE=1;  shift ;;
         --strict)   STRICT=1; shift ;;
@@ -73,24 +78,52 @@ parse_manifest() {
         }
         function emit() {
             if (url != "")
-                printf "%s\037%s\037%s\037%s\037%s\n", url, cat, fn, sha, title
-            url = cat = fn = sha = title = ""
+                printf "%s\037%s\037%s\037%s\037%s\037%s\n", url, cat, fn, sha, title, coll
+            url = cat = fn = sha = title = coll = ""
         }
         /^[[:space:]]*#/ { next }
         /^[[:space:]]*-[[:space:]]*url:/      { emit(); url   = val($0); next }
         /^[[:space:]]+filename:/              { fn    = val($0); next }
         /^[[:space:]]+category:/              { cat   = val($0); next }
+        /^[[:space:]]+collection:/            { coll  = val($0); next }
         /^[[:space:]]+sha256:/                { sha   = val($0); next }
         /^[[:space:]]+title:/                 { title = val($0); next }
         END { emit() }
     ' "$MANIFEST"
 }
 
+# Emits one record per collection definition:  id \037 name \037 description
+parse_collections() {
+    awk '
+        function val(line) {
+            sub(/^[^:]*:[[:space:]]*/, "", line)
+            sub(/[[:space:]]+#.*$/, "", line)
+            gsub(/^[[:space:]]+|[[:space:]]+$/, "", line)
+            gsub(/^"|"$/, "", line)
+            return line
+        }
+        function emit() {
+            if (id != "") printf "%s\037%s\037%s\n", id, name, desc
+            id = name = desc = ""
+        }
+        /^collections:[[:space:]]*$/          { incoll = 1; next }
+        /^sources:[[:space:]]*$/              { if (incoll) { emit(); incoll = 0 } }
+        incoll && /^[[:space:]]*-[[:space:]]*id:/ { emit(); id = val($0); next }
+        incoll && /^[[:space:]]+name:/            { name = val($0); next }
+        incoll && /^[[:space:]]+description:/     { desc = val($0); next }
+        END { emit() }
+    ' "$MANIFEST"
+}
+
 # ── Validate ─────────────────────────────────────────────────────────
 validate_manifest() {
     local errors=0 count=0
-    declare -A seen
-    while IFS="$US" read -r url cat fn sha title; do
+    declare -A seen defined
+    # Collections declared in the top-level `collections:` block
+    while IFS="$US" read -r id name desc; do
+        [[ -n "$id" ]] && defined[$id]=1
+    done < <(parse_collections)
+    while IFS="$US" read -r url cat fn sha title coll; do
         count=$((count + 1))
         local where="entry #$count (${fn:-$url})"
         if [[ ! "$url" =~ ^https?:// ]]; then
@@ -111,6 +144,11 @@ validate_manifest() {
         if [[ -z "$title" ]]; then
             echo "  ✗ $where: missing title"; errors=$((errors+1))
         fi
+        if [[ -z "$coll" ]]; then
+            echo "  ✗ $where: missing collection"; errors=$((errors+1))
+        elif [[ -z "${defined[$coll]:-}" ]]; then
+            echo "  ✗ $where: collection '$coll' not declared in the collections: block"; errors=$((errors+1))
+        fi
         if [[ -n "$sha" && ! "$sha" =~ ^[0-9a-fA-F]{64}$ ]]; then
             echo "  ✗ $where: sha256 must be 64 hex chars"; errors=$((errors+1))
         fi
@@ -128,15 +166,28 @@ validate_manifest() {
 
 # ── List ─────────────────────────────────────────────────────────────
 list_manifest() {
-    printf '%-18s %-52s %s\n' "CATEGORY" "FILENAME" "TITLE"
-    while IFS="$US" read -r url cat fn sha title; do
-        printf '%-18s %-52s %s\n' "$cat" "$fn" "$title"
+    printf '%-22s %-18s %-48s %s\n' "COLLECTION" "CATEGORY" "FILENAME" "TITLE"
+    while IFS="$US" read -r url cat fn sha title coll; do
+        printf '%-22s %-18s %-48s %s\n' "$coll" "$cat" "$fn" "$title"
     done < <(parse_manifest | sort)
 }
 
+# Print the declared collections with a live item count for each.
+list_collections() {
+    declare -A cnt
+    while IFS="$US" read -r url cat fn sha title coll; do
+        [[ -n "$coll" ]] && cnt[$coll]=$(( ${cnt[$coll]:-0} + 1 ))
+    done < <(parse_manifest)
+    printf '%-24s %-6s %s\n' "ID" "ITEMS" "NAME"
+    while IFS="$US" read -r id name desc; do
+        printf '%-24s %-6s %s\n' "$id" "${cnt[$id]:-0}" "$name"
+    done < <(parse_collections)
+}
+
 case "$MODE" in
-    validate) echo "Validating $MANIFEST ..."; validate_manifest; exit $? ;;
-    list)     list_manifest; exit 0 ;;
+    validate)         echo "Validating $MANIFEST ..."; validate_manifest; exit $? ;;
+    list)             list_manifest; exit 0 ;;
+    list-collections) list_collections; exit 0 ;;
 esac
 
 # ── Fetch ────────────────────────────────────────────────────────────
@@ -163,8 +214,9 @@ echo "── Fetching manifest: $MANIFEST → $DEST"
 ok=0; skipped=0; failed=0
 failed_list=""
 
-while IFS="$US" read -r url cat fn sha title; do
-    [[ -n "$ONLY_CATEGORY" && "$cat" != "$ONLY_CATEGORY" ]] && continue
+while IFS="$US" read -r url cat fn sha title coll; do
+    [[ -n "$ONLY_CATEGORY"   && "$cat"  != "$ONLY_CATEGORY"   ]] && continue
+    [[ -n "$ONLY_COLLECTION" && "$coll" != "$ONLY_COLLECTION" ]] && continue
 
     dir="$DEST/$cat"
     out="$dir/$fn"

public/app.js

@@ -294,6 +294,152 @@
     }
   }
 
+  // ── Add-content modal (catalog import + upload) ─────────────────────
+
+  let importPoll = null;
+
+  function openLibraryModal() {
+    $('library-modal').hidden = false;
+    populateUploadCategories();
+    loadCatalog();
+  }
+
+  function closeLibraryModal() {
+    $('library-modal').hidden = true;
+    if (importPoll) { clearInterval(importPoll); importPoll = null; }  // stop background polling
+  }
+
+  async function loadCatalog() {
+    const list = $('catalog-list');
+    list.textContent = 'Loading…';
+    try {
+      const res = await fetch('/api/catalog');
+      if (!res.ok) throw new Error('HTTP ' + res.status);
+      const data = await res.json();
+      renderCatalog(data);
+      if (data.importing) pollImport();   // resume progress polling if an import is already running
+    } catch (e) {
+      list.textContent = 'Could not load the catalog.';
+    }
+  }
+
+  function renderCatalog(data) {
+    const list = $('catalog-list');
+    list.innerHTML = '';
+    (data.collections || []).forEach((c) => {
+      const done = c.count > 0 && c.installed >= c.count;
+      const row = document.createElement('div');
+      row.className = 'catalog-row';
+
+      const meta = document.createElement('div');
+      meta.className = 'catalog-meta';
+      meta.innerHTML =
+        `<span class="catalog-name">${escapeHtml(c.name)}</span>` +
+        `<span class="catalog-desc">${escapeHtml(c.description)}</span>` +
+        `<span class="catalog-count">${c.installed}/${c.count} installed</span>`;
+
+      const btn = document.createElement('button');
+      btn.type = 'button';
+      btn.className = 'ghost-btn catalog-install' + (done ? ' is-done' : '');
+      btn.textContent = done ? 'Installed' : (data.importing ? 'Working…' : 'Install');
+      btn.disabled = done || data.importing || !data.import_enabled;
+      btn.addEventListener('click', () => importCollection(c.id, btn));
+
+      row.appendChild(meta);
+      row.appendChild(btn);
+      list.appendChild(row);
+    });
+
+    if (!data.import_enabled) {
+      const note = document.createElement('p');
+      note.className = 'modal-note';
+      note.textContent = 'Network import is disabled on this deployment — you can still upload your own files below.';
+      list.appendChild(note);
+    }
+  }
+
+  async function importCollection(id, btn) {
+    btn.disabled = true;
+    btn.textContent = 'Starting…';
+    try {
+      const res = await fetch('/api/import', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ collection: id }),
+      });
+      if (res.status === 202) {
+        pollImport();
+      } else {
+        const d = await res.json().catch(() => ({}));
+        btn.textContent = 'Retry';
+        btn.disabled = false;
+        if (res.status === 409) pollImport();
+        console.warn('import:', d.error || res.status);
+      }
+    } catch (e) {
+      btn.textContent = 'Retry';
+      btn.disabled = false;
+    }
+  }
+
+  // While an import runs, refresh the catalog (install progress) and the
+  // sidebar library (docs appear as they download + index), until done.
+  function pollImport() {
+    if (importPoll) return;
+    importPoll = setInterval(async () => {
+      try {
+        const res = await fetch('/api/catalog');
+        if (res.ok) {
+          const data = await res.json();
+          renderCatalog(data);
+          refreshLibrary();
+          if (!data.importing) { clearInterval(importPoll); importPoll = null; }
+        }
+      } catch (e) { /* keep polling */ }
+    }, 3000);
+  }
+
+  function populateUploadCategories() {
+    const sel = $('upload-category');
+    if (sel.options.length) return;
+    Object.keys(CATEGORY_LABELS).forEach((cat) => {
+      const o = document.createElement('option');
+      o.value = cat;
+      o.textContent = CATEGORY_LABELS[cat];
+      sel.appendChild(o);
+    });
+  }
+
+  async function uploadFile(file) {
+    const status = $('upload-status');
+    status.hidden = false;
+    status.className = 'modal-status';
+    status.textContent = 'Uploading ' + file.name + '…';
+    const fd = new FormData();
+    fd.append('category', $('upload-category').value);
+    fd.append('file', file);
+    try {
+      const res = await fetch('/api/upload', { method: 'POST', body: fd });
+      const d = await res.json().catch(() => ({}));
+      if (res.ok) {
+        status.textContent = 'Uploaded ' + (d.filename || file.name) + ' — indexing shortly.';
+        resetUploadPicker();
+        refreshLibrary();
+      } else {
+        status.className = 'modal-status is-error';
+        status.textContent = 'Upload failed: ' + (d.error || ('HTTP ' + res.status));
+      }
+    } catch (e) {
+      status.className = 'modal-status is-error';
+      status.textContent = 'Upload failed.';
+    }
+  }
+
+  function resetUploadPicker() {
+    $('upload-file').value = '';
+    $('upload-drop-text').textContent = 'Choose a PDF or TXT file, or drop it here…';
+  }
+
   // ── Query flow ─────────────────────────────────────────────────────
 
   function setBusy(busy) {
@@ -410,6 +556,44 @@
     $('sidebar-toggle').addEventListener('click', () => toggleSidebar());
     $('backdrop').addEventListener('click', () => toggleSidebar(false));
 
+    // ── Add-content modal wiring ──────────────────────────────────
+    $('library-add-btn').addEventListener('click', openLibraryModal);
+    $('library-modal-close').addEventListener('click', closeLibraryModal);
+    $('library-modal-backdrop').addEventListener('click', closeLibraryModal);
+    document.addEventListener('keydown', (e) => {
+      if (e.key === 'Escape' && !$('library-modal').hidden) closeLibraryModal();
+    });
+
+    const fileInput = $('upload-file');
+    const drop = $('upload-drop');
+    drop.addEventListener('click', () => fileInput.click());
+    fileInput.addEventListener('change', () => {
+      if (fileInput.files.length) $('upload-drop-text').textContent = fileInput.files[0].name;
+    });
+    ['dragover', 'dragenter'].forEach((ev) =>
+      drop.addEventListener(ev, (e) => { e.preventDefault(); drop.classList.add('dragover'); }));
+    ['dragleave', 'dragend'].forEach((ev) =>
+      drop.addEventListener(ev, () => drop.classList.remove('dragover')));
+    drop.addEventListener('drop', (e) => {
+      e.preventDefault();
+      drop.classList.remove('dragover');
+      if (e.dataTransfer.files.length) {
+        fileInput.files = e.dataTransfer.files;
+        $('upload-drop-text').textContent = e.dataTransfer.files[0].name;
+      }
+    });
+    $('upload-form').addEventListener('submit', (e) => {
+      e.preventDefault();
+      if (fileInput.files.length) {
+        uploadFile(fileInput.files[0]);
+      } else {
+        const s = $('upload-status');
+        s.hidden = false;
+        s.className = 'modal-status';
+        s.textContent = 'Pick a file first.';
+      }
+    });
+
     $('user-input').focus();
 
     refreshStatus();

public/index.html

@@ -39,7 +39,10 @@ <h2 class="panel-title">System</h2>
       </section>
 
       <section class="panel panel-grow" aria-label="Field library">
-        <h2 class="panel-title">Field library <span class="badge" id="library-count">0</span></h2>
+        <h2 class="panel-title">Field library <span class="badge" id="library-count">0</span>
+          <button class="ghost-btn panel-action" id="library-add-btn" type="button"
+                  aria-haspopup="dialog">+ Add</button>
+        </h2>
         <div class="library" id="library-list">
           <p class="library-empty">No documents indexed yet. Run the content
           fetcher or drop PDFs into the sources volume — they are indexed
@@ -96,6 +99,42 @@ <h2 class="panel-title">Field library <span class="badge" id="library-count">0</
     </main>
   </div>
 
+  <!-- ── Add-content modal ─────────────────────────────────────── -->
+  <div class="modal" id="library-modal" hidden role="dialog" aria-modal="true"
+       aria-labelledby="library-modal-title">
+    <div class="modal-backdrop" id="library-modal-backdrop"></div>
+    <div class="modal-card" role="document">
+      <header class="modal-head">
+        <h2 id="library-modal-title">Add to the field library</h2>
+        <button class="icon-btn" id="library-modal-close" type="button" aria-label="Close">
+          <svg viewBox="0 0 24 24" aria-hidden="true"><path d="M6 6l12 12M18 6L6 18" stroke="currentColor" stroke-width="2" stroke-linecap="round" fill="none"/></svg>
+        </button>
+      </header>
+      <p class="modal-sub">Import curated public-domain collections, or upload your own
+        PDF/TXT documents. New files are indexed automatically — no restart needed.</p>
+
+      <section class="modal-section">
+        <h3 class="modal-h3">Curated collections</h3>
+        <div id="catalog-list" class="catalog-list">Loading…</div>
+      </section>
+
+      <section class="modal-section">
+        <h3 class="modal-h3">Upload your own</h3>
+        <form id="upload-form" class="upload-form">
+          <label class="upload-cat">Category
+            <select id="upload-category"></select>
+          </label>
+          <button type="button" class="upload-drop" id="upload-drop">
+            <input type="file" id="upload-file" accept=".pdf,.txt" hidden>
+            <span id="upload-drop-text">Choose a PDF or TXT file, or drop it here…</span>
+          </button>
+          <button type="submit" class="ghost-btn" id="upload-submit">Upload</button>
+        </form>
+        <p class="modal-status" id="upload-status" hidden></p>
+      </section>
+    </div>
+  </div>
+
   <script src="app.js"></script>
 </body>
 </html>