diff --git a/helper-scripts/fetch-source-data.sh b/helper-scripts/fetch-source-data.sh index cbca9b9..d29fee2 100755 --- a/helper-scripts/fetch-source-data.sh +++ b/helper-scripts/fetch-source-data.sh @@ -16,7 +16,9 @@ # --dest DIR library directory (default: ./public/sources) # --seed DIR copy DIR/* into the library first (no clobber) # --category CAT only fetch one category (e.g. 200_Medical) +# --collection ID only fetch one collection (e.g. austere-medicine) # --list print the manifest and exit +# --list-collections print the declared collections and exit # --validate lint the manifest (no network) and exit # --force re-download files that already exist # --strict exit non-zero if any download failed @@ -31,6 +33,7 @@ MANIFEST="sources.yaml" DEST="public/sources" SEED="" ONLY_CATEGORY="" +ONLY_COLLECTION="" MODE="fetch" FORCE=0 STRICT=0 @@ -41,7 +44,9 @@ while [[ $# -gt 0 ]]; do --dest) DEST="$2"; shift 2 ;; --seed) SEED="$2"; shift 2 ;; --category) ONLY_CATEGORY="$2"; shift 2 ;; + --collection) ONLY_COLLECTION="$2"; shift 2 ;; --list) MODE="list"; shift ;; + --list-collections) MODE="list-collections"; shift ;; --validate) MODE="validate"; shift ;; --force) FORCE=1; shift ;; --strict) STRICT=1; shift ;; @@ -73,24 +78,52 @@ parse_manifest() { } function emit() { if (url != "") - printf "%s\037%s\037%s\037%s\037%s\n", url, cat, fn, sha, title - url = cat = fn = sha = title = "" + printf "%s\037%s\037%s\037%s\037%s\037%s\n", url, cat, fn, sha, title, coll + url = cat = fn = sha = title = coll = "" } /^[[:space:]]*#/ { next } /^[[:space:]]*-[[:space:]]*url:/ { emit(); url = val($0); next } /^[[:space:]]+filename:/ { fn = val($0); next } /^[[:space:]]+category:/ { cat = val($0); next } + /^[[:space:]]+collection:/ { coll = val($0); next } /^[[:space:]]+sha256:/ { sha = val($0); next } /^[[:space:]]+title:/ { title = val($0); next } END { emit() } ' "$MANIFEST" } +# Emits one record per collection definition: id \037 name \037 description +parse_collections() { + awk ' + function val(line) { + sub(/^[^:]*:[[:space:]]*/, "", line) + sub(/[[:space:]]+#.*$/, "", line) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", line) + gsub(/^"|"$/, "", line) + return line + } + function emit() { + if (id != "") printf "%s\037%s\037%s\n", id, name, desc + id = name = desc = "" + } + /^collections:[[:space:]]*$/ { incoll = 1; next } + /^sources:[[:space:]]*$/ { if (incoll) { emit(); incoll = 0 } } + incoll && /^[[:space:]]*-[[:space:]]*id:/ { emit(); id = val($0); next } + incoll && /^[[:space:]]+name:/ { name = val($0); next } + incoll && /^[[:space:]]+description:/ { desc = val($0); next } + END { emit() } + ' "$MANIFEST" +} + # ── Validate ───────────────────────────────────────────────────────── validate_manifest() { local errors=0 count=0 - declare -A seen - while IFS="$US" read -r url cat fn sha title; do + declare -A seen defined + # Collections declared in the top-level `collections:` block + while IFS="$US" read -r id name desc; do + [[ -n "$id" ]] && defined[$id]=1 + done < <(parse_collections) + while IFS="$US" read -r url cat fn sha title coll; do count=$((count + 1)) local where="entry #$count (${fn:-$url})" if [[ ! "$url" =~ ^https?:// ]]; then @@ -111,6 +144,11 @@ validate_manifest() { if [[ -z "$title" ]]; then echo " ✗ $where: missing title"; errors=$((errors+1)) fi + if [[ -z "$coll" ]]; then + echo " ✗ $where: missing collection"; errors=$((errors+1)) + elif [[ -z "${defined[$coll]:-}" ]]; then + echo " ✗ $where: collection '$coll' not declared in the collections: block"; errors=$((errors+1)) + fi if [[ -n "$sha" && ! "$sha" =~ ^[0-9a-fA-F]{64}$ ]]; then echo " ✗ $where: sha256 must be 64 hex chars"; errors=$((errors+1)) fi @@ -128,15 +166,28 @@ validate_manifest() { # ── List ───────────────────────────────────────────────────────────── list_manifest() { - printf '%-18s %-52s %s\n' "CATEGORY" "FILENAME" "TITLE" - while IFS="$US" read -r url cat fn sha title; do - printf '%-18s %-52s %s\n' "$cat" "$fn" "$title" + printf '%-22s %-18s %-48s %s\n' "COLLECTION" "CATEGORY" "FILENAME" "TITLE" + while IFS="$US" read -r url cat fn sha title coll; do + printf '%-22s %-18s %-48s %s\n' "$coll" "$cat" "$fn" "$title" done < <(parse_manifest | sort) } +# Print the declared collections with a live item count for each. +list_collections() { + declare -A cnt + while IFS="$US" read -r url cat fn sha title coll; do + [[ -n "$coll" ]] && cnt[$coll]=$(( ${cnt[$coll]:-0} + 1 )) + done < <(parse_manifest) + printf '%-24s %-6s %s\n' "ID" "ITEMS" "NAME" + while IFS="$US" read -r id name desc; do + printf '%-24s %-6s %s\n' "$id" "${cnt[$id]:-0}" "$name" + done < <(parse_collections) +} + case "$MODE" in - validate) echo "Validating $MANIFEST ..."; validate_manifest; exit $? ;; - list) list_manifest; exit 0 ;; + validate) echo "Validating $MANIFEST ..."; validate_manifest; exit $? ;; + list) list_manifest; exit 0 ;; + list-collections) list_collections; exit 0 ;; esac # ── Fetch ──────────────────────────────────────────────────────────── @@ -163,8 +214,9 @@ echo "── Fetching manifest: $MANIFEST → $DEST" ok=0; skipped=0; failed=0 failed_list="" -while IFS="$US" read -r url cat fn sha title; do - [[ -n "$ONLY_CATEGORY" && "$cat" != "$ONLY_CATEGORY" ]] && continue +while IFS="$US" read -r url cat fn sha title coll; do + [[ -n "$ONLY_CATEGORY" && "$cat" != "$ONLY_CATEGORY" ]] && continue + [[ -n "$ONLY_COLLECTION" && "$coll" != "$ONLY_COLLECTION" ]] && continue dir="$DEST/$cat" out="$dir/$fn" diff --git a/public/app.js b/public/app.js index 7a39024..fb8813d 100644 --- a/public/app.js +++ b/public/app.js @@ -294,6 +294,152 @@ } } + // ── Add-content modal (catalog import + upload) ───────────────────── + + let importPoll = null; + + function openLibraryModal() { + $('library-modal').hidden = false; + populateUploadCategories(); + loadCatalog(); + } + + function closeLibraryModal() { + $('library-modal').hidden = true; + if (importPoll) { clearInterval(importPoll); importPoll = null; } // stop background polling + } + + async function loadCatalog() { + const list = $('catalog-list'); + list.textContent = 'Loading…'; + try { + const res = await fetch('/api/catalog'); + if (!res.ok) throw new Error('HTTP ' + res.status); + const data = await res.json(); + renderCatalog(data); + if (data.importing) pollImport(); // resume progress polling if an import is already running + } catch (e) { + list.textContent = 'Could not load the catalog.'; + } + } + + function renderCatalog(data) { + const list = $('catalog-list'); + list.innerHTML = ''; + (data.collections || []).forEach((c) => { + const done = c.count > 0 && c.installed >= c.count; + const row = document.createElement('div'); + row.className = 'catalog-row'; + + const meta = document.createElement('div'); + meta.className = 'catalog-meta'; + meta.innerHTML = + `${escapeHtml(c.name)}` + + `${escapeHtml(c.description)}` + + `${c.installed}/${c.count} installed`; + + const btn = document.createElement('button'); + btn.type = 'button'; + btn.className = 'ghost-btn catalog-install' + (done ? ' is-done' : ''); + btn.textContent = done ? 'Installed' : (data.importing ? 'Working…' : 'Install'); + btn.disabled = done || data.importing || !data.import_enabled; + btn.addEventListener('click', () => importCollection(c.id, btn)); + + row.appendChild(meta); + row.appendChild(btn); + list.appendChild(row); + }); + + if (!data.import_enabled) { + const note = document.createElement('p'); + note.className = 'modal-note'; + note.textContent = 'Network import is disabled on this deployment — you can still upload your own files below.'; + list.appendChild(note); + } + } + + async function importCollection(id, btn) { + btn.disabled = true; + btn.textContent = 'Starting…'; + try { + const res = await fetch('/api/import', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ collection: id }), + }); + if (res.status === 202) { + pollImport(); + } else { + const d = await res.json().catch(() => ({})); + btn.textContent = 'Retry'; + btn.disabled = false; + if (res.status === 409) pollImport(); + console.warn('import:', d.error || res.status); + } + } catch (e) { + btn.textContent = 'Retry'; + btn.disabled = false; + } + } + + // While an import runs, refresh the catalog (install progress) and the + // sidebar library (docs appear as they download + index), until done. + function pollImport() { + if (importPoll) return; + importPoll = setInterval(async () => { + try { + const res = await fetch('/api/catalog'); + if (res.ok) { + const data = await res.json(); + renderCatalog(data); + refreshLibrary(); + if (!data.importing) { clearInterval(importPoll); importPoll = null; } + } + } catch (e) { /* keep polling */ } + }, 3000); + } + + function populateUploadCategories() { + const sel = $('upload-category'); + if (sel.options.length) return; + Object.keys(CATEGORY_LABELS).forEach((cat) => { + const o = document.createElement('option'); + o.value = cat; + o.textContent = CATEGORY_LABELS[cat]; + sel.appendChild(o); + }); + } + + async function uploadFile(file) { + const status = $('upload-status'); + status.hidden = false; + status.className = 'modal-status'; + status.textContent = 'Uploading ' + file.name + '…'; + const fd = new FormData(); + fd.append('category', $('upload-category').value); + fd.append('file', file); + try { + const res = await fetch('/api/upload', { method: 'POST', body: fd }); + const d = await res.json().catch(() => ({})); + if (res.ok) { + status.textContent = 'Uploaded ' + (d.filename || file.name) + ' — indexing shortly.'; + resetUploadPicker(); + refreshLibrary(); + } else { + status.className = 'modal-status is-error'; + status.textContent = 'Upload failed: ' + (d.error || ('HTTP ' + res.status)); + } + } catch (e) { + status.className = 'modal-status is-error'; + status.textContent = 'Upload failed.'; + } + } + + function resetUploadPicker() { + $('upload-file').value = ''; + $('upload-drop-text').textContent = 'Choose a PDF or TXT file, or drop it here…'; + } + // ── Query flow ───────────────────────────────────────────────────── function setBusy(busy) { @@ -410,6 +556,44 @@ $('sidebar-toggle').addEventListener('click', () => toggleSidebar()); $('backdrop').addEventListener('click', () => toggleSidebar(false)); + // ── Add-content modal wiring ────────────────────────────────── + $('library-add-btn').addEventListener('click', openLibraryModal); + $('library-modal-close').addEventListener('click', closeLibraryModal); + $('library-modal-backdrop').addEventListener('click', closeLibraryModal); + document.addEventListener('keydown', (e) => { + if (e.key === 'Escape' && !$('library-modal').hidden) closeLibraryModal(); + }); + + const fileInput = $('upload-file'); + const drop = $('upload-drop'); + drop.addEventListener('click', () => fileInput.click()); + fileInput.addEventListener('change', () => { + if (fileInput.files.length) $('upload-drop-text').textContent = fileInput.files[0].name; + }); + ['dragover', 'dragenter'].forEach((ev) => + drop.addEventListener(ev, (e) => { e.preventDefault(); drop.classList.add('dragover'); })); + ['dragleave', 'dragend'].forEach((ev) => + drop.addEventListener(ev, () => drop.classList.remove('dragover'))); + drop.addEventListener('drop', (e) => { + e.preventDefault(); + drop.classList.remove('dragover'); + if (e.dataTransfer.files.length) { + fileInput.files = e.dataTransfer.files; + $('upload-drop-text').textContent = e.dataTransfer.files[0].name; + } + }); + $('upload-form').addEventListener('submit', (e) => { + e.preventDefault(); + if (fileInput.files.length) { + uploadFile(fileInput.files[0]); + } else { + const s = $('upload-status'); + s.hidden = false; + s.className = 'modal-status'; + s.textContent = 'Pick a file first.'; + } + }); + $('user-input').focus(); refreshStatus(); diff --git a/public/index.html b/public/index.html index b0443f7..2e0b8c8 100644 --- a/public/index.html +++ b/public/index.html @@ -39,7 +39,10 @@

System

-

Field library 0

+

Field library 0 + +

No documents indexed yet. Run the content fetcher or drop PDFs into the sources volume — they are indexed @@ -96,6 +99,42 @@

Field library 0

+ + + diff --git a/public/style.css b/public/style.css index 700b308..276fbf6 100644 --- a/public/style.css +++ b/public/style.css @@ -765,3 +765,121 @@ body { .toggle span { display: none; } } + +/* ══════════════════════════════════════════════════════════════════════ + Add-content: Field-library action + import/upload modal + ══════════════════════════════════════════════════════════════════════ */ + +.panel-action { + margin-left: auto; + padding: 2px 8px; + font-size: 0.68rem; + letter-spacing: 0.04em; +} + +.modal[hidden] { display: none; } +.modal { + position: fixed; + inset: 0; + z-index: 50; + display: flex; + align-items: center; + justify-content: center; + padding: 20px; +} +.modal-backdrop { + position: absolute; + inset: 0; + background: rgba(2, 16, 17, 0.72); + backdrop-filter: blur(2px); +} +.modal-card { + position: relative; + width: min(640px, 100%); + max-height: 88vh; + overflow-y: auto; + background: var(--bg1); + border: 1px solid var(--line); + border-radius: var(--radius-lg); + box-shadow: var(--shadow-lg); + padding: 20px 22px 24px; +} +.modal-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} +.modal-head h2 { margin: 0; font-size: 1.05rem; } +.modal-sub { + margin: 8px 0 4px; + color: var(--fg-dim); + font-size: 0.85rem; + line-height: 1.5; +} +.modal-section { margin-top: 20px; } +.modal-h3 { + margin: 0 0 10px; + font-family: var(--mono); + font-size: 0.7rem; + font-weight: 600; + letter-spacing: 0.14em; + text-transform: uppercase; + color: var(--fg-dim); +} +.modal-note { margin: 10px 2px 0; font-size: 0.78rem; color: var(--warning); } +.modal-status { margin: 10px 2px 0; font-size: 0.82rem; color: var(--fg-dim); } +.modal-status.is-error { color: var(--err); } + +.catalog-list { display: flex; flex-direction: column; gap: 8px; } +.catalog-row { + display: flex; + align-items: center; + gap: 12px; + background: var(--bg2); + border: 1px solid var(--line); + border-radius: var(--radius-md); + padding: 10px 12px; +} +.catalog-meta { display: flex; flex-direction: column; gap: 2px; flex: 1; min-width: 0; } +.catalog-name { font-weight: 600; font-size: 0.9rem; } +.catalog-desc { color: var(--fg-dim); font-size: 0.78rem; line-height: 1.4; } +.catalog-count { color: var(--fg-faint); font-size: 0.72rem; font-family: var(--mono); } +.catalog-install { flex-shrink: 0; } +.catalog-install.is-done { color: var(--success); border-color: transparent; } + +.upload-form { display: grid; grid-template-columns: 1fr; gap: 10px; } +.upload-cat { + display: flex; + align-items: center; + gap: 8px; + font-size: 0.8rem; + color: var(--fg-dim); +} +.upload-cat select { + flex: 1; + background: var(--bg2); + color: var(--fg); + border: 1px solid var(--line); + border-radius: var(--radius-sm); + padding: 6px 8px; + font-family: inherit; + font-size: 0.82rem; +} +.upload-drop { + display: block; + width: 100%; + text-align: center; + padding: 18px; + background: var(--bg2); + border: 1px dashed var(--fg-faint); + border-radius: var(--radius-md); + color: var(--fg-dim); + font-size: 0.85rem; + cursor: pointer; +} +.upload-drop.dragover { + border-color: var(--accent); + background: var(--accent-soft); + color: var(--fg); +} diff --git a/sources.yaml b/sources.yaml index b04762e..d68f72a 100644 --- a/sources.yaml +++ b/sources.yaml @@ -24,12 +24,43 @@ # Validate after editing: helper-scripts/fetch-source-data.sh --validate # ══════════════════════════════════════════════════════════════════════ +# ── Collections ─────────────────────────────────────────────────────── +# Named bundles a user can import as a unit (see GET /api/catalog and +# `fetch-source-data.sh --collection `). Every item below tags exactly +# one collection via its `collection:` field. +collections: + - id: core-survival + name: "Core Survival & Preparedness" + description: "Field survival, shelter, navigation, weather, and disaster-readiness checklists." + - id: austere-medicine + name: "Austere & Emergency Medicine" + description: "First aid, field surgery, dentistry, and health care where there is no doctor." + - id: food-and-water + name: "Food & Water" + description: "Growing, canning, and preserving food; safe water, gravity-flow systems, solar disinfection." + - id: power-and-engineering + name: "Power & Engineering" + description: "Off-grid solar power and appropriate-technology village engineering." + - id: comms-and-radio + name: "Communications & Radio" + description: "Amateur/emergency radio and interoperable field communications." + - id: education + name: "Education & Textbooks" + description: "Open textbooks and foundational learning material." + - id: civics-and-culture + name: "Civics & Culture" + description: "Founding civic documents and enduring public-domain texts." + - id: software-and-tech + name: "Software & Technical Reference" + description: "Programming, operating systems, and version control references." + sources: # ── 100 Survival & Preparedness ─────────────────────────────────── - url: https://www.ready.gov/sites/default/files/2021-02/ready_checklist.pdf filename: FEMA-Emergency-Checklist.pdf category: 100_Survival + collection: core-survival title: "FEMA — Emergency Supply Checklist" license: PD (US Government) size: ~150 KB @@ -37,6 +68,7 @@ sources: - url: https://www.ready.gov/sites/default/files/2020-03/ready_emergency-financial-first-aid-toolkit.pdf filename: FEMA-Financial-First-Aid-Kit.pdf category: 100_Survival + collection: core-survival title: "FEMA — Emergency Financial First Aid Kit" license: PD (US Government) size: ~1.7 MB @@ -44,6 +76,7 @@ sources: - url: https://archive.org/download/Fm21-76SurvivalManual/FM21-76_SurvivalManual.pdf filename: FM21-76-US-Army-Survival.pdf category: 100_Survival + collection: core-survival title: "US Army Survival Manual FM 21-76 (shelter, water, foraging, trapping)" license: PD (US Government) size: ~3 MB @@ -51,6 +84,7 @@ sources: - url: https://archive.org/download/milmanual-fm-3-25.26-map-reading-and-land-navigation/fm_3-25.26_map_reading_and_land_navigation.pdf filename: FM3-25-26-Map-Reading-Land-Navigation.pdf category: 100_Survival + collection: core-survival title: "US Army FM 3-25.26 — Map Reading and Land Navigation" license: PD (US Government) size: ~24 MB @@ -58,6 +92,7 @@ sources: - url: https://archive.org/download/pdfy-ULdTVDxCZR1mXh25/Nuclear%20War%20Survival%20Skills%20-%20Cresson%20H%20Kearney.pdf filename: NuclearWarSurvivalSkills-CressonHKearney.pdf category: 100_Survival + collection: core-survival title: "Nuclear War Survival Skills (Kearney, Oak Ridge National Laboratory)" license: PD (US Government research; author authorized free reproduction) size: ~6.4 MB @@ -65,6 +100,7 @@ sources: - url: https://www.weather.gov/media/owlie/SGJune6-11.pdf filename: NWS-Weather-Spotters-Field-Guide.pdf category: 100_Survival + collection: core-survival title: "NOAA/NWS — Weather Spotter's Field Guide" license: PD (US Government) size: ~28 MB @@ -72,21 +108,24 @@ sources: - url: https://www.gutenberg.org/cache/epub/17093/pg17093.txt filename: Camp-Life-In-The-Woods-Trapping.txt category: 100_Survival + collection: core-survival title: "Camp Life in the Woods and the Tricks of Trapping (Gibson, 1881)" - license: PD (Project Gutenberg #17093) + license: "PD (Project Gutenberg #17093)" size: ~500 KB - url: https://www.gutenberg.org/cache/epub/28255/pg28255.txt filename: Shelters-Shacks-And-Shanties.txt category: 100_Survival + collection: core-survival title: "Shelters, Shacks and Shanties (D.C. Beard, 1914)" - license: PD (Project Gutenberg #28255) + license: "PD (Project Gutenberg #28255)" size: ~400 KB # ── 200 Medical ─────────────────────────────────────────────────── - url: https://archive.org/download/where-there-is-no-doctor/Where%20There%20Is%20No%20Doctor.pdf filename: Where-There-Is-No-Doctor.pdf category: 200_Medical + collection: austere-medicine title: "Where There Is No Doctor (Hesperian)" license: Hesperian open copyright (free non-commercial reproduction) size: ~8.7 MB @@ -94,6 +133,7 @@ sources: - url: https://archive.org/download/where-there-is-no-dentist-hesperian-2010/Where_there_is_no_dentist_-_Hesperian_2010.pdf filename: Where-There-Is-No-Dentist.pdf category: 200_Medical + collection: austere-medicine title: "Where There Is No Dentist (Hesperian)" license: Hesperian open copyright (free non-commercial reproduction) size: ~6.8 MB @@ -101,6 +141,7 @@ sources: - url: https://archive.org/download/WhereWomenHaveNoDoctor/18.HersperianFoundation-WhereWomenHaveNoDoctor.pdf filename: Where-Women-Have-No-Doctor.pdf category: 200_Medical + collection: austere-medicine title: "Where Women Have No Doctor (Hesperian)" license: Hesperian open copyright (free non-commercial reproduction) size: ~12 MB @@ -108,6 +149,7 @@ sources: - url: https://archive.org/download/UsArmyFirstAidManualFm4-25.11/UsArmyFirstAidManual.pdf filename: FM4-25-11-First-Aid.pdf category: 200_Medical + collection: austere-medicine title: "US Army FM 4-25.11 — First Aid" license: PD (US Government) size: ~2.6 MB @@ -115,6 +157,7 @@ sources: - url: https://archive.org/download/emergency-war-surgery-5th-edition/Emergency%20War%20Surgery%205th%20Edition.pdf filename: Emergency-War-Surgery-5th-Edition.pdf category: 200_Medical + collection: austere-medicine title: "Emergency War Surgery, 5th Edition (Borden Institute)" license: PD (US Government) size: ~30 MB @@ -122,6 +165,7 @@ sources: - url: https://archive.org/download/survival-and-austere-medicine-3rd-ed-2017/Survival%20and%20Austere%20Medicine%203rd%20Ed%20%282017%29.pdf filename: Survival-And-Austere-Medicine-3rd-Ed.pdf category: 200_Medical + collection: austere-medicine title: "Survival and Austere Medicine: An Introduction (3rd ed., 2017)" license: Free distribution (authors permit non-commercial copying) size: ~23 MB @@ -129,6 +173,7 @@ sources: - url: https://www.nctsn.org/sites/default/files/resources/pfa_field_operations_guide.pdf filename: PFA-Field-Operations-Guide.pdf category: 200_Medical + collection: austere-medicine title: "Psychological First Aid — Field Operations Guide (NCTSN/NCPTSD)" license: Free distribution (US Government funded) size: ~3.4 MB @@ -137,6 +182,7 @@ sources: - url: https://archive.org/download/usda-guide-home-canning/USDA-Complete-Guide-to-Home-Canning-2015-revision.pdf filename: USDA-Complete-Guide-Home-Canning.pdf category: 300_Food + collection: food-and-water title: "USDA Complete Guide to Home Canning (2015 revision, all 7 guides)" license: PD (US Government) size: ~17 MB @@ -144,14 +190,16 @@ sources: - url: https://www.gutenberg.org/cache/epub/9550/pg9550.txt filename: Manual-Of-Gardening-Bailey.txt category: 300_Food + collection: food-and-water title: "Manual of Gardening, 2nd ed. (L.H. Bailey)" - license: PD (Project Gutenberg #9550) + license: "PD (Project Gutenberg #9550)" size: ~900 KB # ── 400 Engineering, Water & Power ──────────────────────────────── - url: https://www.nrel.gov/docs/fy04osti/36178.pdf filename: NREL-PV-Solar-Systems.pdf category: 400_Engineering + collection: power-and-engineering title: "NREL — Photovoltaic Solar Resource of the United States" license: PD (US Government) size: ~1 MB @@ -159,6 +207,7 @@ sources: - url: https://archive.org/download/VillageTechnologyHandbook/Village%20Technology%20Handbook.pdf filename: VITA-Village-Technology-Handbook.pdf category: 400_Engineering + collection: power-and-engineering title: "VITA — Village Technology Handbook (water, agriculture, construction)" license: PD / freely distributed (USAID-funded) size: ~15 MB @@ -166,6 +215,7 @@ sources: - url: https://archive.org/download/fa_Handbook_of_Gravity-Flow_Water_Systems/Handbook_of_Gravity-Flow_Water_Systems.pdf filename: Handbook-Of-Gravity-Flow-Water-Systems.pdf category: 400_Engineering + collection: food-and-water title: "A Handbook of Gravity-Flow Water Systems (Jordan)" license: Freely distributed (Appropriate Technology Library) size: ~18 MB @@ -173,6 +223,7 @@ sources: - url: https://archive.org/download/fa_Solar_Disinfection_of_Water/Solar_Disinfection_of_Water.pdf filename: Solar-Disinfection-Of-Water.pdf category: 400_Engineering + collection: food-and-water title: "Solar Disinfection of Water (SODIS)" license: Freely distributed (Appropriate Technology Library) size: ~13 MB @@ -181,6 +232,7 @@ sources: - url: https://www.arrl.org/files/file/Technology/tis/info/pdf/0209038.pdf filename: ARRL-Emergency-Comms.pdf category: 500_Comms + collection: comms-and-radio title: "ARRL — Emergency Communications" license: Freely distributed (ARRL) size: ~540 KB @@ -188,6 +240,7 @@ sources: - url: https://archive.org/download/national-interoperability-field-operations-guide-nifog-version-2.02/National%20Interoperability%20Field%20Operations%20Guide%20%28NIFOG%29%20Version%202.02.pdf filename: CISA-NIFOG-2.02.pdf category: 500_Comms + collection: comms-and-radio title: "CISA — National Interoperability Field Operations Guide (NIFOG) v2.02" license: PD (US Government) size: ~6.7 MB @@ -196,6 +249,7 @@ sources: - url: https://assets.openstax.org/oscms-prodcms/media/documents/ElementaryAlgebra2e-WEB.pdf filename: OpenStax-Elementary-Algebra-2e.pdf category: 600_Education + collection: education title: "OpenStax — Elementary Algebra 2e" license: CC BY 4.0 size: ~45 MB @@ -204,6 +258,7 @@ sources: - url: https://www.un.org/sites/un2.un.org/files/2021/03/udhr.pdf filename: UN-Universal-Declaration-Human-Rights.pdf category: 700_Social + collection: civics-and-culture title: "Universal Declaration of Human Rights" license: PD (United Nations) size: ~100 KB @@ -211,6 +266,7 @@ sources: - url: https://www.govinfo.gov/content/pkg/CDOC-110hdoc50/pdf/CDOC-110hdoc50.pdf filename: US-Constitution.pdf category: 700_Social + collection: civics-and-culture title: "The Constitution of the United States (pocket edition)" license: PD (US Government) size: ~4 MB @@ -218,21 +274,24 @@ sources: - url: https://www.gutenberg.org/cache/epub/1404/pg1404.txt filename: Federalist-Papers.txt category: 700_Social + collection: civics-and-culture title: "The Federalist Papers" - license: PD (Project Gutenberg #1404) + license: "PD (Project Gutenberg #1404)" size: ~1.2 MB - url: https://www.gutenberg.org/cache/epub/10/pg10.txt filename: KJV-Bible.txt category: 700_Social + collection: civics-and-culture title: "King James Bible" - license: PD (Project Gutenberg #10) + license: "PD (Project Gutenberg #10)" size: ~4.4 MB # ── 800 Software / Technical Reference ──────────────────────────── - url: https://greenteapress.com/thinkpython2/thinkpython2.pdf filename: ThinkPython2.pdf category: 800_Software + collection: software-and-tech title: "Think Python, 2nd Edition (Downey)" license: CC BY-NC size: ~920 KB @@ -240,6 +299,7 @@ sources: - url: https://greenteapress.com/thinkos/thinkos.pdf filename: ThinkOS.pdf category: 800_Software + collection: software-and-tech title: "Think OS: A Brief Introduction to Operating Systems (Downey)" license: CC BY-NC size: ~370 KB @@ -247,6 +307,7 @@ sources: - url: https://eloquentjavascript.net/Eloquent_JavaScript.pdf filename: Eloquent-JavaScript.pdf category: 800_Software + collection: software-and-tech title: "Eloquent JavaScript, 3rd Edition (Haverbeke)" license: CC BY-NC size: ~2 MB @@ -254,6 +315,7 @@ sources: - url: https://github.com/progit/progit2/releases/download/2.1.426/progit.pdf filename: Pro-Git-2.pdf category: 800_Software + collection: software-and-tech title: "Pro Git, 2nd Edition (Chacon & Straub)" license: CC BY-NC-SA size: ~19 MB diff --git a/src/catalog.h b/src/catalog.h new file mode 100644 index 0000000..6ea7298 --- /dev/null +++ b/src/catalog.h @@ -0,0 +1,115 @@ +#pragma once + +// ── Catalog parser ─────────────────────────────────────────────────── +// Reads the curated source manifest (sources.yaml) into memory. This is a +// deliberately minimal line parser for the flat schema that manifest uses — +// NOT a general YAML parser — and mirrors the awk parser in +// helper-scripts/fetch-source-data.sh so the two never diverge. The manifest +// is a trusted, image-baked file, so the parser optimises for clarity. + +#include +#include +#include +#include "config.h" + +struct CatalogItem { + std::string url, filename, category, collection, title, license, size, sha256; +}; + +struct CatalogCollection { + std::string id, name, description; +}; + +struct Catalog { + std::vector collections; + std::vector items; + + const CatalogCollection* find_collection(const std::string& id) const { + for (const auto& c : collections) if (c.id == id) return &c; + return nullptr; + } +}; + +// Clean a raw value that follows the first ':' on a line. A double-quoted +// value is returned verbatim (so a legitimate '#' inside it — e.g. a license +// like "PD (Project Gutenberg #17093)" — is preserved). An unquoted value has +// a trailing " # comment" stripped and is trimmed. +inline std::string catalog_clean_value(std::string v) { + size_t b = v.find_first_not_of(" \t"); + if (b == std::string::npos) return ""; + v = v.substr(b); + + if (v.front() == '"') { + size_t end = v.find('"', 1); + if (end != std::string::npos) return v.substr(1, end - 1); + // Unterminated quote — fall through to best-effort unquoted handling. + } + + size_t h = v.find(" #"); + if (h != std::string::npos) v = v.substr(0, h); + size_t e = v.find_last_not_of(" \t\r"); + return (e == std::string::npos) ? "" : v.substr(0, e + 1); +} + +inline Catalog parse_catalog(const std::string& path) { + Catalog cat; + std::ifstream in(path); + if (!in) return cat; + + enum Section { NONE, COLLECTIONS, SOURCES } sec = NONE; + CatalogItem item; bool have_item = false; + CatalogCollection col; bool have_col = false; + + auto flush_item = [&] { + if (have_item) cat.items.push_back(item); + item = CatalogItem(); have_item = false; + }; + auto flush_col = [&] { + if (have_col) cat.collections.push_back(col); + col = CatalogCollection(); have_col = false; + }; + + std::string line; + while (std::getline(in, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + size_t b = line.find_first_not_of(" \t"); + if (b == std::string::npos) continue; // blank + std::string t = line.substr(b); + if (t[0] == '#') continue; // comment + + // Top-level keys (no indentation) switch sections. + if (b == 0) { + if (t.rfind("collections:", 0) == 0) { flush_col(); flush_item(); sec = COLLECTIONS; } + else if (t.rfind("sources:", 0) == 0) { flush_col(); flush_item(); sec = SOURCES; } + else { flush_col(); flush_item(); sec = NONE; } + continue; + } + + // Indented "key: value" (optionally list-prefixed with "- "). + bool dash = false; + std::string s = t; + if (s.rfind("- ", 0) == 0) { dash = true; s = s.substr(2); } + size_t colon = s.find(':'); + if (colon == std::string::npos) continue; + std::string key = s.substr(0, colon); + std::string val = catalog_clean_value(s.substr(colon + 1)); + + if (sec == COLLECTIONS) { + if (dash && key == "id") { flush_col(); col.id = val; have_col = true; } + else if (key == "name") col.name = val; + else if (key == "description") col.description = val; + } else if (sec == SOURCES) { + if (dash && key == "url") { flush_item(); item.url = val; have_item = true; } + else if (key == "filename") item.filename = val; + else if (key == "category") item.category = val; + else if (key == "collection") item.collection = val; + else if (key == "title") item.title = val; + else if (key == "license") item.license = val; + else if (key == "size") item.size = val; + else if (key == "sha256") item.sha256 = val; + } + } + flush_col(); + flush_item(); + return cat; +} diff --git a/src/config.h b/src/config.h index 9d5fd5e..0ded8e4 100644 --- a/src/config.h +++ b/src/config.h @@ -34,6 +34,14 @@ const size_t MAX_QUERY_CHARS = 8000; // longest accepted question const size_t MAX_CONV_ID_CHARS = 128; const size_t MAX_CONVERSATIONS = 200; // bounded history map +// Largest single file accepted by POST /api/upload. cpp-httplib's payload +// limit is global, so this also caps how much ANY request can buffer — kept +// modest to bound that exposure. Large curated documents are pulled by the +// server-side importer (curl), not through an upload body, so this only needs +// to fit a user's own document; the /query and /api/import handlers separately +// reject bodies over MAX_REQUEST_BODY. +const size_t MAX_UPLOAD_BYTES = 32u * 1024u * 1024u; // 32 MB per uploaded file + // ── Ingestion ──────────────────────────────────────────────────────── const size_t MAX_DOCUMENT_CHARS = 8u * 1000u * 1000u; // per-document text cap const int FILE_SETTLE_SECONDS = 10; // skip files modified more recently @@ -167,3 +175,22 @@ inline std::string get_llm_model_name() { inline std::string get_embedding_model_name() { return env_or("EMBEDDING_MODEL", "nomic-embed-text"); } + +// ── Content import (catalog + upload) ───────────────────────────────── +// The curated manifest the /api/catalog + /api/import endpoints read, and the +// fetcher script they exec to download vetted catalog items. Both are baked +// into the image (see Dockerfile); overridable for local runs and tests. +inline std::string get_catalog_path() { + return env_or("JIC_CATALOG_PATH", "sources.yaml"); +} + +inline std::string get_fetcher_path() { + return env_or("JIC_FETCHER_PATH", "/app/bin/fetch-sources"); +} + +// Catalog import reaches out to the network. Allow locking it down for +// air-gapped deployments (uploading local files stays available regardless). +inline bool network_import_enabled() { + const char* v = getenv("JIC_DISABLE_NETWORK_IMPORT"); + return !(v && *v); +} diff --git a/src/server.cpp b/src/server.cpp index 8eef77e..950a103 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -18,6 +18,10 @@ #include #include #include +#include +#include +#include +#include #include "httplib.h" #include "llama.h" @@ -28,6 +32,7 @@ #include "embeddings.h" #include "llm.h" #include "sqlite_vec_index.h" +#include "catalog.h" namespace fs = std::filesystem; using json = nlohmann::json; @@ -127,6 +132,10 @@ static void handle_query(const httplib::Request& req, httplib::Response& res) { } // ── Parse & validate input (client errors → 400, not 500) ─────── + if (req.body.size() > MAX_REQUEST_BODY) { // /query is small JSON, not an upload + send_error(res, 413, "Request body too large"); + return; + } json rj; try { rj = json::parse(req.body); @@ -326,6 +335,287 @@ static void handle_library(const httplib::Request&, httplib::Response& res) { res.set_content(out.dump(), "application/json"); } +// ═════════════════════════════════════════════════════════════════════ +// Content import & upload +// +// Two ways to grow the library at runtime, both writing into the sources +// volume where the ingestion service auto-indexes new files: +// • POST /api/import — download a vetted catalog collection. The server +// execs the bundled fetcher, which only ever contacts the fixed URLs in +// the image-baked manifest, so there is no arbitrary-URL / SSRF surface. +// • POST /api/upload — accept a user's own PDF/TXT (works fully offline). +// GET /api/catalog lists what is available and what is already installed. +// ═════════════════════════════════════════════════════════════════════ + +extern char** environ; + +// One import at a time — a collection download can take minutes; this guards +// against a client spawning many concurrent fetchers. +static std::atomic g_import_running{false}; + +static bool str_ends_with(const std::string& s, const std::string& suf) { + return s.size() >= suf.size() && + s.compare(s.size() - suf.size(), suf.size(), suf) == 0; +} + +// Reduce a browser-supplied filename to a safe basename: no path components, +// no traversal, a conservative character set. Returns "" if nothing usable. +static std::string sanitize_upload_filename(const std::string& raw) { + std::string name = fs::path(raw).filename().string(); // strip any path + std::string out; + for (char c : name) { + unsigned char u = static_cast(c); + if (std::isalnum(u) || c == '.' || c == '_' || c == '-' || + c == ' ' || c == '(' || c == ')') + out += c; + } + while (!out.empty() && (out.front() == ' ' || out.front() == '.')) + out.erase(out.begin()); + while (!out.empty() && out.back() == ' ') + out.pop_back(); + if (out.empty() || out.size() > 200) return ""; + if (out.find("..") != std::string::npos) return ""; + return out; +} + +// Category must match the taxonomy pattern NNN_Alpha (e.g. 200_Medical); this +// alone forbids path separators and traversal. +static bool valid_upload_category(const std::string& c) { + if (c.size() < 5 || c.size() > 40) return false; + if (!std::isdigit((unsigned char)c[0]) || !std::isdigit((unsigned char)c[1]) || + !std::isdigit((unsigned char)c[2]) || c[3] != '_') return false; + for (size_t i = 4; i < c.size(); i++) + if (!std::isalpha((unsigned char)c[i])) return false; + return true; +} + +// Download one catalog collection by exec-ing the bundled fetcher. Uses +// posix_spawn with an explicit argv (never a shell), so the collection id — +// already validated against the catalog — cannot be interpreted as a command. +// Runs on a detached worker thread; clears the single-flight flag when done. +static void run_import_collection(std::string collection) { + std::string fetcher = get_fetcher_path(); + std::string manifest = get_catalog_path(); + std::string dest = get_sources_dir(); + + std::vector args = { + fetcher, "--manifest", manifest, "--dest", dest, + "--collection", collection + }; + std::vector argv; + for (auto& a : args) argv.push_back(const_cast(a.c_str())); + argv.push_back(nullptr); + + std::cout << "Import: fetching collection '" << collection << "'" << std::endl; + pid_t pid = 0; + int rc = posix_spawn(&pid, fetcher.c_str(), nullptr, nullptr, + argv.data(), environ); + if (rc == 0) { + int status = 0; + waitpid(pid, &status, 0); + std::cout << "Import: collection '" << collection << "' done (exit " + << (WIFEXITED(status) ? WEXITSTATUS(status) : -1) << ")" << std::endl; + } else { + std::cerr << "Import: failed to spawn fetcher (" << get_fetcher_path() + << "): " << std::strerror(rc) << std::endl; + } + g_import_running.store(false); +} + +// GET /api/catalog — the importable catalog, grouped by collection, with an +// "installed" flag per item (file already present in the sources volume). +static void handle_catalog(const httplib::Request&, httplib::Response& res) { + Catalog cat = parse_catalog(get_catalog_path()); + const fs::path sources_root = get_sources_dir(); + + json cols = json::array(); + for (const auto& c : cat.collections) { + json items = json::array(); + int installed = 0; + for (const auto& it : cat.items) { + if (it.collection != c.id) continue; + std::error_code ec; + bool present = fs::exists(sources_root / it.category / it.filename, ec) && !ec; + if (present) installed++; + items.push_back({ + {"filename", it.filename}, + {"category", it.category}, + {"title", it.title}, + {"license", it.license}, + {"size", it.size}, + {"installed", present} + }); + } + cols.push_back({ + {"id", c.id}, + {"name", c.name}, + {"description", c.description}, + {"count", items.size()}, + {"installed", installed}, + {"items", items} + }); + } + json out; + out["collections"] = cols; + out["import_enabled"] = network_import_enabled(); + out["importing"] = g_import_running.load(); + // error_handler_t::replace: never throw out of the handler if a manifest + // string contains an invalid UTF-8 byte — emit a replacement char instead. + res.set_content(out.dump(-1, ' ', false, json::error_handler_t::replace), + "application/json"); +} + +// POST /api/import { "collection": "" } — start downloading a collection. +static void handle_import(const httplib::Request& req, httplib::Response& res) { + if (!network_import_enabled()) { + send_error(res, 403, "Catalog import over the network is disabled on this deployment."); + return; + } + if (req.body.size() > MAX_REQUEST_BODY) { // this endpoint needs only tiny JSON + send_error(res, 413, "Request body too large"); + return; + } + json rj; + try { rj = json::parse(req.body); } + catch (const std::exception&) { send_error(res, 400, "Request body must be valid JSON"); return; } + + if (!rj.contains("collection") || !rj["collection"].is_string()) { + send_error(res, 400, "Missing required string field: collection"); + return; + } + std::string collection = rj["collection"]; + + // Validate against the catalog (a closed set) AND as a strict slug — + // defence in depth, even though posix_spawn never involves a shell. + Catalog cat = parse_catalog(get_catalog_path()); + if (!cat.find_collection(collection)) { + send_error(res, 404, "Unknown collection id"); + return; + } + for (char c : collection) + if (!(std::islower((unsigned char)c) || std::isdigit((unsigned char)c) || c == '-')) { + send_error(res, 400, "Invalid collection id"); + return; + } + + bool expected = false; + if (!g_import_running.compare_exchange_strong(expected, true)) { + send_error(res, 409, "An import is already in progress."); + return; + } + try { + std::thread(run_import_collection, collection).detach(); + } catch (const std::exception& e) { + g_import_running.store(false); // never leave the flag wedged + std::cerr << "Import: could not start worker: " << e.what() << std::endl; + send_error(res, 503, "Could not start the import worker; please retry."); + return; + } + + res.status = 202; + json out; + out["status"] = "started"; + out["collection"] = collection; + res.set_content(out.dump(), "application/json"); +} + +// POST /api/upload (multipart: file=, category=NNN_Name) +static void handle_upload(const httplib::Request& req, httplib::Response& res) { + if (!req.is_multipart_form_data()) { + send_error(res, 400, "Expected multipart/form-data"); + return; + } + if (!req.has_file("file")) { // clean 400 rather than risk a throw on a missing field + send_error(res, 400, "Missing form field: file"); + return; + } + auto file = req.get_file_value("file"); + std::string category = req.has_file("category") + ? req.get_file_value("category").content + : (req.has_param("category") ? req.get_param_value("category") : ""); + + if (!valid_upload_category(category)) { + send_error(res, 400, "Invalid category (expected NNN_Name, e.g. 200_Medical)"); + return; + } + std::string fn = sanitize_upload_filename(file.filename); + if (fn.empty()) { send_error(res, 400, "Invalid or missing filename"); return; } + + // Validate the extension case-insensitively (ingestion lowercases it, so + // e.g. FOO.PDF is a valid PDF and must pass the same gate here). + std::string ext_lc = fn; + for (char& c : ext_lc) c = static_cast(std::tolower((unsigned char)c)); + const bool is_pdf = str_ends_with(ext_lc, ".pdf"); + const bool is_txt = str_ends_with(ext_lc, ".txt"); + if (!is_pdf && !is_txt) { + send_error(res, 400, "Only .pdf or .txt files can be indexed"); + return; + } + if (file.content.empty()) { send_error(res, 400, "Empty file"); return; } + if (file.content.size() > MAX_UPLOAD_BYTES) { + send_error(res, 413, "File exceeds the size limit"); + return; + } + if (is_pdf && file.content.compare(0, 4, "%PDF") != 0) { + send_error(res, 400, "File does not look like a PDF"); + return; + } + + std::error_code ec; + fs::path dir = fs::path(get_sources_dir()) / category; + fs::create_directories(dir, ec); + if (ec) { send_error(res, 500, "Could not create the category directory"); return; } + + // Refuse when the volume is nearly full, so uploads can't fill the disk and + // wedge the search index / DB. Fails CLOSED: if free space can't even be + // queried, reject rather than write blindly. + std::error_code space_ec; + auto space = fs::space(dir, space_ec); + if (space_ec || space.available < file.content.size() + (64ull << 20)) { + send_error(res, 507, "Not enough free space in the library volume"); + return; + } + + fs::path target = dir / fn; + if (fs::exists(target, ec)) { + send_error(res, 409, "A file with that name already exists in this category"); + return; + } + // Unique temp name so two concurrent uploads never write the same .part + // (the ".part" suffix is not an ingestible extension, so a transient temp + // is never indexed). + static std::atomic upload_seq{0}; + fs::path part = dir / (fn + "." + std::to_string(upload_seq.fetch_add(1)) + ".part"); + { + std::ofstream o(part, std::ios::binary | std::ios::trunc); + if (!o) { send_error(res, 500, "Could not write the file"); return; } + o.write(file.content.data(), static_cast(file.content.size())); + if (!o.good()) { o.close(); fs::remove(part, ec); send_error(res, 500, "Write failed"); return; } + } + // Finalise atomically WITHOUT clobbering: link() fails with EEXIST if the + // target appeared meanwhile, closing the check-then-rename race. The + // ingester never sees the .part (not an ingestible extension). + if (link(part.c_str(), target.c_str()) != 0) { + int e = errno; + fs::remove(part, ec); + if (e == EEXIST) { + send_error(res, 409, "A file with that name already exists in this category"); + } else { + send_error(res, 500, "Could not finalise the file"); + } + return; + } + fs::remove(part, ec); // drop the temporary hard link; target remains + + std::cout << "Upload: stored " << category << "/" << fn + << " (" << file.content.size() << " bytes)" << std::endl; + json out; + out["status"] = "ok"; + out["category"] = category; + out["filename"] = fn; + res.set_content(out.dump(), "application/json"); +} + // ═════════════════════════════════════════════════════════════════════ // Background thread: refresh cached counts every 10 s // ═════════════════════════════════════════════════════════════════════ @@ -503,8 +793,10 @@ int main() { httplib::Server svr; g_server = &svr; - // Reject oversized request bodies before buffering them - svr.set_payload_max_length(MAX_REQUEST_BODY); + // Body cap. Raised to MAX_UPLOAD_BYTES so /api/upload can accept real + // documents; cpp-httplib's limit is global, and on this single-user + // appliance that upper bound on buffered request size is acceptable. + svr.set_payload_max_length(MAX_UPLOAD_BYTES); svr.set_read_timeout(15, 0); svr.set_write_timeout(60, 0); @@ -546,6 +838,9 @@ int main() { svr.Post("/query", handle_query); svr.Get ("/status", handle_status); svr.Get ("/api/library", handle_library); + svr.Get ("/api/catalog", handle_catalog); + svr.Post("/api/import", handle_import); + svr.Post("/api/upload", handle_upload); svr.Options(".*", [&cors_origin](const httplib::Request&, httplib::Response& res) { res.status = cors_origin.empty() ? 405 : 204; });