diff --git a/helper-scripts/fetch-source-data.sh b/helper-scripts/fetch-source-data.sh index cbca9b9..d29fee2 100755 --- a/helper-scripts/fetch-source-data.sh +++ b/helper-scripts/fetch-source-data.sh @@ -16,7 +16,9 @@ # --dest DIR library directory (default: ./public/sources) # --seed DIR copy DIR/* into the library first (no clobber) # --category CAT only fetch one category (e.g. 200_Medical) +# --collection ID only fetch one collection (e.g. austere-medicine) # --list print the manifest and exit +# --list-collections print the declared collections and exit # --validate lint the manifest (no network) and exit # --force re-download files that already exist # --strict exit non-zero if any download failed @@ -31,6 +33,7 @@ MANIFEST="sources.yaml" DEST="public/sources" SEED="" ONLY_CATEGORY="" +ONLY_COLLECTION="" MODE="fetch" FORCE=0 STRICT=0 @@ -41,7 +44,9 @@ while [[ $# -gt 0 ]]; do --dest) DEST="$2"; shift 2 ;; --seed) SEED="$2"; shift 2 ;; --category) ONLY_CATEGORY="$2"; shift 2 ;; + --collection) ONLY_COLLECTION="$2"; shift 2 ;; --list) MODE="list"; shift ;; + --list-collections) MODE="list-collections"; shift ;; --validate) MODE="validate"; shift ;; --force) FORCE=1; shift ;; --strict) STRICT=1; shift ;; @@ -73,24 +78,52 @@ parse_manifest() { } function emit() { if (url != "") - printf "%s\037%s\037%s\037%s\037%s\n", url, cat, fn, sha, title - url = cat = fn = sha = title = "" + printf "%s\037%s\037%s\037%s\037%s\037%s\n", url, cat, fn, sha, title, coll + url = cat = fn = sha = title = coll = "" } /^[[:space:]]*#/ { next } /^[[:space:]]*-[[:space:]]*url:/ { emit(); url = val($0); next } /^[[:space:]]+filename:/ { fn = val($0); next } /^[[:space:]]+category:/ { cat = val($0); next } + /^[[:space:]]+collection:/ { coll = val($0); next } /^[[:space:]]+sha256:/ { sha = val($0); next } /^[[:space:]]+title:/ { title = val($0); next } END { emit() } ' "$MANIFEST" } +# Emits one record per collection definition: id \037 name \037 description +parse_collections() { + awk ' + function val(line) { + sub(/^[^:]*:[[:space:]]*/, "", line) + sub(/[[:space:]]+#.*$/, "", line) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", line) + gsub(/^"|"$/, "", line) + return line + } + function emit() { + if (id != "") printf "%s\037%s\037%s\n", id, name, desc + id = name = desc = "" + } + /^collections:[[:space:]]*$/ { incoll = 1; next } + /^sources:[[:space:]]*$/ { if (incoll) { emit(); incoll = 0 } } + incoll && /^[[:space:]]*-[[:space:]]*id:/ { emit(); id = val($0); next } + incoll && /^[[:space:]]+name:/ { name = val($0); next } + incoll && /^[[:space:]]+description:/ { desc = val($0); next } + END { emit() } + ' "$MANIFEST" +} + # ── Validate ───────────────────────────────────────────────────────── validate_manifest() { local errors=0 count=0 - declare -A seen - while IFS="$US" read -r url cat fn sha title; do + declare -A seen defined + # Collections declared in the top-level `collections:` block + while IFS="$US" read -r id name desc; do + [[ -n "$id" ]] && defined[$id]=1 + done < <(parse_collections) + while IFS="$US" read -r url cat fn sha title coll; do count=$((count + 1)) local where="entry #$count (${fn:-$url})" if [[ ! "$url" =~ ^https?:// ]]; then @@ -111,6 +144,11 @@ validate_manifest() { if [[ -z "$title" ]]; then echo " ✗ $where: missing title"; errors=$((errors+1)) fi + if [[ -z "$coll" ]]; then + echo " ✗ $where: missing collection"; errors=$((errors+1)) + elif [[ -z "${defined[$coll]:-}" ]]; then + echo " ✗ $where: collection '$coll' not declared in the collections: block"; errors=$((errors+1)) + fi if [[ -n "$sha" && ! "$sha" =~ ^[0-9a-fA-F]{64}$ ]]; then echo " ✗ $where: sha256 must be 64 hex chars"; errors=$((errors+1)) fi @@ -128,15 +166,28 @@ validate_manifest() { # ── List ───────────────────────────────────────────────────────────── list_manifest() { - printf '%-18s %-52s %s\n' "CATEGORY" "FILENAME" "TITLE" - while IFS="$US" read -r url cat fn sha title; do - printf '%-18s %-52s %s\n' "$cat" "$fn" "$title" + printf '%-22s %-18s %-48s %s\n' "COLLECTION" "CATEGORY" "FILENAME" "TITLE" + while IFS="$US" read -r url cat fn sha title coll; do + printf '%-22s %-18s %-48s %s\n' "$coll" "$cat" "$fn" "$title" done < <(parse_manifest | sort) } +# Print the declared collections with a live item count for each. +list_collections() { + declare -A cnt + while IFS="$US" read -r url cat fn sha title coll; do + [[ -n "$coll" ]] && cnt[$coll]=$(( ${cnt[$coll]:-0} + 1 )) + done < <(parse_manifest) + printf '%-24s %-6s %s\n' "ID" "ITEMS" "NAME" + while IFS="$US" read -r id name desc; do + printf '%-24s %-6s %s\n' "$id" "${cnt[$id]:-0}" "$name" + done < <(parse_collections) +} + case "$MODE" in - validate) echo "Validating $MANIFEST ..."; validate_manifest; exit $? ;; - list) list_manifest; exit 0 ;; + validate) echo "Validating $MANIFEST ..."; validate_manifest; exit $? ;; + list) list_manifest; exit 0 ;; + list-collections) list_collections; exit 0 ;; esac # ── Fetch ──────────────────────────────────────────────────────────── @@ -163,8 +214,9 @@ echo "── Fetching manifest: $MANIFEST → $DEST" ok=0; skipped=0; failed=0 failed_list="" -while IFS="$US" read -r url cat fn sha title; do - [[ -n "$ONLY_CATEGORY" && "$cat" != "$ONLY_CATEGORY" ]] && continue +while IFS="$US" read -r url cat fn sha title coll; do + [[ -n "$ONLY_CATEGORY" && "$cat" != "$ONLY_CATEGORY" ]] && continue + [[ -n "$ONLY_COLLECTION" && "$coll" != "$ONLY_COLLECTION" ]] && continue dir="$DEST/$cat" out="$dir/$fn" diff --git a/public/app.js b/public/app.js index 7a39024..fb8813d 100644 --- a/public/app.js +++ b/public/app.js @@ -294,6 +294,152 @@ } } + // ── Add-content modal (catalog import + upload) ───────────────────── + + let importPoll = null; + + function openLibraryModal() { + $('library-modal').hidden = false; + populateUploadCategories(); + loadCatalog(); + } + + function closeLibraryModal() { + $('library-modal').hidden = true; + if (importPoll) { clearInterval(importPoll); importPoll = null; } // stop background polling + } + + async function loadCatalog() { + const list = $('catalog-list'); + list.textContent = 'Loading…'; + try { + const res = await fetch('/api/catalog'); + if (!res.ok) throw new Error('HTTP ' + res.status); + const data = await res.json(); + renderCatalog(data); + if (data.importing) pollImport(); // resume progress polling if an import is already running + } catch (e) { + list.textContent = 'Could not load the catalog.'; + } + } + + function renderCatalog(data) { + const list = $('catalog-list'); + list.innerHTML = ''; + (data.collections || []).forEach((c) => { + const done = c.count > 0 && c.installed >= c.count; + const row = document.createElement('div'); + row.className = 'catalog-row'; + + const meta = document.createElement('div'); + meta.className = 'catalog-meta'; + meta.innerHTML = + `${escapeHtml(c.name)}` + + `${escapeHtml(c.description)}` + + `${c.installed}/${c.count} installed`; + + const btn = document.createElement('button'); + btn.type = 'button'; + btn.className = 'ghost-btn catalog-install' + (done ? ' is-done' : ''); + btn.textContent = done ? 'Installed' : (data.importing ? 'Working…' : 'Install'); + btn.disabled = done || data.importing || !data.import_enabled; + btn.addEventListener('click', () => importCollection(c.id, btn)); + + row.appendChild(meta); + row.appendChild(btn); + list.appendChild(row); + }); + + if (!data.import_enabled) { + const note = document.createElement('p'); + note.className = 'modal-note'; + note.textContent = 'Network import is disabled on this deployment — you can still upload your own files below.'; + list.appendChild(note); + } + } + + async function importCollection(id, btn) { + btn.disabled = true; + btn.textContent = 'Starting…'; + try { + const res = await fetch('/api/import', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ collection: id }), + }); + if (res.status === 202) { + pollImport(); + } else { + const d = await res.json().catch(() => ({})); + btn.textContent = 'Retry'; + btn.disabled = false; + if (res.status === 409) pollImport(); + console.warn('import:', d.error || res.status); + } + } catch (e) { + btn.textContent = 'Retry'; + btn.disabled = false; + } + } + + // While an import runs, refresh the catalog (install progress) and the + // sidebar library (docs appear as they download + index), until done. + function pollImport() { + if (importPoll) return; + importPoll = setInterval(async () => { + try { + const res = await fetch('/api/catalog'); + if (res.ok) { + const data = await res.json(); + renderCatalog(data); + refreshLibrary(); + if (!data.importing) { clearInterval(importPoll); importPoll = null; } + } + } catch (e) { /* keep polling */ } + }, 3000); + } + + function populateUploadCategories() { + const sel = $('upload-category'); + if (sel.options.length) return; + Object.keys(CATEGORY_LABELS).forEach((cat) => { + const o = document.createElement('option'); + o.value = cat; + o.textContent = CATEGORY_LABELS[cat]; + sel.appendChild(o); + }); + } + + async function uploadFile(file) { + const status = $('upload-status'); + status.hidden = false; + status.className = 'modal-status'; + status.textContent = 'Uploading ' + file.name + '…'; + const fd = new FormData(); + fd.append('category', $('upload-category').value); + fd.append('file', file); + try { + const res = await fetch('/api/upload', { method: 'POST', body: fd }); + const d = await res.json().catch(() => ({})); + if (res.ok) { + status.textContent = 'Uploaded ' + (d.filename || file.name) + ' — indexing shortly.'; + resetUploadPicker(); + refreshLibrary(); + } else { + status.className = 'modal-status is-error'; + status.textContent = 'Upload failed: ' + (d.error || ('HTTP ' + res.status)); + } + } catch (e) { + status.className = 'modal-status is-error'; + status.textContent = 'Upload failed.'; + } + } + + function resetUploadPicker() { + $('upload-file').value = ''; + $('upload-drop-text').textContent = 'Choose a PDF or TXT file, or drop it here…'; + } + // ── Query flow ───────────────────────────────────────────────────── function setBusy(busy) { @@ -410,6 +556,44 @@ $('sidebar-toggle').addEventListener('click', () => toggleSidebar()); $('backdrop').addEventListener('click', () => toggleSidebar(false)); + // ── Add-content modal wiring ────────────────────────────────── + $('library-add-btn').addEventListener('click', openLibraryModal); + $('library-modal-close').addEventListener('click', closeLibraryModal); + $('library-modal-backdrop').addEventListener('click', closeLibraryModal); + document.addEventListener('keydown', (e) => { + if (e.key === 'Escape' && !$('library-modal').hidden) closeLibraryModal(); + }); + + const fileInput = $('upload-file'); + const drop = $('upload-drop'); + drop.addEventListener('click', () => fileInput.click()); + fileInput.addEventListener('change', () => { + if (fileInput.files.length) $('upload-drop-text').textContent = fileInput.files[0].name; + }); + ['dragover', 'dragenter'].forEach((ev) => + drop.addEventListener(ev, (e) => { e.preventDefault(); drop.classList.add('dragover'); })); + ['dragleave', 'dragend'].forEach((ev) => + drop.addEventListener(ev, () => drop.classList.remove('dragover'))); + drop.addEventListener('drop', (e) => { + e.preventDefault(); + drop.classList.remove('dragover'); + if (e.dataTransfer.files.length) { + fileInput.files = e.dataTransfer.files; + $('upload-drop-text').textContent = e.dataTransfer.files[0].name; + } + }); + $('upload-form').addEventListener('submit', (e) => { + e.preventDefault(); + if (fileInput.files.length) { + uploadFile(fileInput.files[0]); + } else { + const s = $('upload-status'); + s.hidden = false; + s.className = 'modal-status'; + s.textContent = 'Pick a file first.'; + } + }); + $('user-input').focus(); refreshStatus(); diff --git a/public/index.html b/public/index.html index b0443f7..2e0b8c8 100644 --- a/public/index.html +++ b/public/index.html @@ -39,7 +39,10 @@
No documents indexed yet. Run the content fetcher or drop PDFs into the sources volume — they are indexed @@ -96,6 +99,42 @@
Import curated public-domain collections, or upload your own + PDF/TXT documents. New files are indexed automatically — no restart needed.
+ +