diff --git a/README.md b/README.md index 652d5cd..7b1f5a5 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,11 @@ so those engines see no cookies/identifier from you, and keeps anything it store Marginalia, Mwmbl, and Wikipedia** (plus optional bring-your-own **Brave**, **Mojeek**, and **Kagi** API keys), then de-duplicated and re-ranked. The app proxies the requests: upstream engines see no cookies, no referrer, no user/device identifier, and a rotated User-Agent. It **never scrapes Google**. +- **Search operators.** The query syntax you already know: `"exact phrase"`, `-term`, `site:` / + `-site:`, `intitle:`, `inurl:`, `filetype:` (or `ext:`), `before:` / `after:` dates, and `OR`. + Operators the upstream engines understand are forwarded to them; all structural filters are also + enforced on-device over the merged results, so they work consistently across every engine. The + served home page has a collapsible cheat sheet. - **Typo and "similar sounding" tolerance.** A misspelled query surfaces a "Did you mean" suggestion, from the engines' own correction when offered, otherwise from a fully on-device corrector (phonetic + edit-distance over a bundled dictionary, enriched by your own history). No new outbound calls. diff --git a/app/src/main/java/org/searchmob/engine/EngineAdapter.kt b/app/src/main/java/org/searchmob/engine/EngineAdapter.kt index d3bf53a..61b27e2 100644 --- a/app/src/main/java/org/searchmob/engine/EngineAdapter.kt +++ b/app/src/main/java/org/searchmob/engine/EngineAdapter.kt @@ -10,6 +10,8 @@ enum class SearchCategory { GENERAL, } data class SearchQuery( val terms: String, val category: SearchCategory = SearchCategory.GENERAL, + /** Operator-free text used for on-device relevance ranking; defaults to [terms]. */ + val rankingTerms: String = terms, ) /** A normalized result from one engine, with the rank (0-based position) it held in that engine's list. */ diff --git a/app/src/main/java/org/searchmob/engine/MetaSearchResultProvider.kt b/app/src/main/java/org/searchmob/engine/MetaSearchResultProvider.kt index e23dc7e..b9ebb54 100644 --- a/app/src/main/java/org/searchmob/engine/MetaSearchResultProvider.kt +++ b/app/src/main/java/org/searchmob/engine/MetaSearchResultProvider.kt @@ -10,6 +10,7 @@ import org.searchmob.engine.aggregate.EngineOutcome import org.searchmob.engine.correct.NoopSpellCorrector import org.searchmob.engine.correct.SpellCorrector import org.searchmob.engine.http.HttpClientFactory +import org.searchmob.engine.query.QueryOperators import org.searchmob.engine.rank.DomainRanker import org.searchmob.engine.rank.PersonalizationModel import org.searchmob.engine.rank.Personalizer @@ -27,11 +28,18 @@ import org.searchmob.server.SearchResultProvider * Bridges the metasearch engine to the server's [SearchResultProvider] contract, so the engine drops * in behind the unchanged HTTP routes. * + * The raw query is run through [QueryOperators] before anything else: Google-fu style operators + * (`site:`, `-exclude`, `"phrase"`, `intitle:`/`inurl:`, `filetype:`/`ext:`, `before:`/`after:`, + * `OR`/`|`) are split off into an operator-free `cleanText` (for relevance/correction/the contextual + * summary) and an `engineQuery` sent upstream; whatever no engine can be trusted to honor is enforced + * locally as a post-filter over the aggregated results. + * * It also surfaces a "did you mean" correction: it prefers the consensus correction the upstream * engines reported (parsed from the responses already fetched, no extra request) and otherwise falls - * back to the offline [corrector]. When the original query returns results, the correction is offered - * as a non-binding suggestion; when it returns nothing and a confident correction exists, the - * correction is searched automatically and reported via [SearchOutcome.showingResultsFor]. + * back to the offline [corrector] (skipped entirely for an operator-laden query, whose syntax the + * corrector would just mangle). When the original query returns results, the correction is offered as + * a non-binding suggestion; when it returns nothing and a confident correction exists, the correction + * is searched automatically and reported via [SearchOutcome.showingResultsFor]. */ class MetaSearchResultProvider( private val registryProvider: suspend () -> EngineRegistry, @@ -95,8 +103,13 @@ class MetaSearchResultProvider( coroutineScope { if (query.isBlank()) return@coroutineScope SearchOutcome(emptyList()) - // Fetch the contextual summary concurrently with the metasearch so it adds no latency. - val summaryDeferred = async { runCatching { summaryFetcher(query) }.getOrNull() } + // Google-fu style operators (site:/intitle:/before:/etc., see QueryOperators) are parsed + // once here so the operator-free text can drive anything that reasons about "what is the + // user asking about" rather than "how do I fetch it". + val parsed = QueryOperators.parse(query) + // Fetch the contextual summary concurrently with the metasearch so it adds no latency. Uses + // the operator-free text: an operator-laden query would never resolve a Wikipedia entity. + val summaryDeferred = async { runCatching { summaryFetcher(parsed.cleanText) }.getOrNull() } // An inline `+name` scope token (parsed by the route) overrides the saved active scope // for this one search only; the persisted selection is never written. val rules = @@ -120,7 +133,12 @@ class MetaSearchResultProvider( ) val upstreamCorrection = upstreamRaw?.takeIf { !it.equals(query, ignoreCase = true) } - val onDevice = corrector.suggest(query) + // The on-device corrector reasons about English word spelling, not query syntax: an + // operator-laden query (parsed.engineQuery differing from parsed.cleanText, or any + // locally-enforced filter) would just get its operators mangled, so it is skipped and the + // suggestion relies on whatever the upstream engines themselves reported. + val hasOperators = parsed.engineQuery != parsed.cleanText || parsed.hasFilters + val onDevice = if (hasOperators) null else corrector.suggest(query) val suggestion = upstreamCorrection ?: onDevice?.corrected?.takeIf { !it.equals(query, ignoreCase = true) } val summary = summaryDeferred.await() @@ -178,7 +196,10 @@ class MetaSearchResultProvider( } } - /** Aggregate [query], sort, apply [rules] locally; return (results, correction, per-engine status). */ + /** + * Aggregate [query] (parsing any Google-fu operators it contains — see [QueryOperators]), sort, + * apply [rules] locally; return (results, correction, per-engine status). + */ private suspend fun aggregateRanked( query: String, vertical: Vertical, @@ -192,20 +213,35 @@ class MetaSearchResultProvider( // rules (pin/raise/block still win); null disables promotion. summaryDeferred: Deferred? = null, ): Triple, String?, List> { - // Scope the query for the chosen vertical (a `site:` OR group the engines understand). The - // original query still drives sort/summary/correction so freshness keywords are detected. - val scoped = Verticals.transformQuery(query, vertical) + val parsed = QueryOperators.parse(query) + // Scope the query for the chosen vertical (a `site:` OR group the engines understand), on top + // of the already-parsed engine query (operators forwarded as-is; intitle:/inurl: turned into a + // bare recall hint). The operator-free text still drives sort/summary/correction so freshness + // keywords are detected without a `site:` clause throwing that off. + val scoped = Verticals.transformQuery(parsed.engineQuery, vertical) // Tailor results to the active UI language (region-neutral when English/unset). val region = languageRegionFor(languageProvider()) val aggregated = - aggregator.aggregate(SearchQuery(scoped), registryProvider().activeEngines(httpClient, region)) + aggregator.aggregate( + SearchQuery(scoped, rankingTerms = parsed.cleanText), + registryProvider().activeEngines(httpClient, region), + ) + // Operators the engines cannot be trusted to honor (intitle:/inurl:/filetype:/before:/after:/ + // -exclusions/site: scoping) are enforced locally here, before sort/personalization/rules see + // the list. + val filtered = + if (parsed.hasFilters) { + aggregated.results.filter { parsed.matches(it.title, it.url, it.snippet, it.publishedMillis) } + } else { + aggregated.results + } // Sort first (relevance/date/freshness blend), then nudge by the owner's learned click model // (between sort and rules, so PIN/RAISE/BLOCK still win), then bucket by rules. val sorted = ResultSorter.sort( - aggregated.results, + filtered, sortMode, - query, + parsed.cleanText, System.currentTimeMillis(), relevanceOf = { it.score }, publishedOf = { it.publishedMillis }, @@ -215,7 +251,7 @@ class MetaSearchResultProvider( Personalizer.reorder( sorted, { DomainRanker.host(it.url) }, - query, + parsed.cleanText, personalization, System.currentTimeMillis(), ) diff --git a/app/src/main/java/org/searchmob/engine/aggregate/Aggregator.kt b/app/src/main/java/org/searchmob/engine/aggregate/Aggregator.kt index 82edc6f..13e02b9 100644 --- a/app/src/main/java/org/searchmob/engine/aggregate/Aggregator.kt +++ b/app/src/main/java/org/searchmob/engine/aggregate/Aggregator.kt @@ -97,7 +97,10 @@ class Aggregator( else -> EngineOutcome(adapter.id, EngineStatus.FAILED) } } - val results = rank(successes.flatMap { it.items }, query.terms) + // rankingTerms is the operator-free text (site:/filetype:/etc. clauses stripped, see + // QueryOperators) so a scoping clause never pollutes lexical relevance; the correction check + // still compares against the full terms actually sent upstream. + val results = rank(successes.flatMap { it.items }, query.rankingTerms) return AggregationResult(results, consensusCorrection(query.terms, successes), engineStatus) } diff --git a/app/src/main/java/org/searchmob/engine/query/QueryOperators.kt b/app/src/main/java/org/searchmob/engine/query/QueryOperators.kt new file mode 100644 index 0000000..a52bffe --- /dev/null +++ b/app/src/main/java/org/searchmob/engine/query/QueryOperators.kt @@ -0,0 +1,353 @@ +package org.searchmob.engine.query + +import org.searchmob.engine.rank.DomainRanker +import java.net.URI +import java.time.LocalDate +import java.time.ZoneOffset + +/** + * A query parsed into its Google-fu style operators plus the leftover free text. + * + * [engineQuery] is what actually goes upstream: operators the engines themselves understand + * (`site:`, quoted phrases, `-exclusions`, `OR`/`|`) are forwarded verbatim so the engine's own index + * does the heavy lifting; operators no public engine implements consistently (`intitle:`, `inurl:`, + * `before:`, `after:`) are either turned into a plain recall hint or dropped, and the actual constraint + * is enforced locally by [matches] over the aggregated results. [cleanText] is the query with every + * operator stripped to plain words, for anything that reasons about "what is the user asking about" + * rather than "how do I fetch it" (on-device relevance, spelling correction, the contextual-summary + * lookup). + */ +data class ParsedQuery( + val raw: String, + val cleanText: String, + val engineQuery: String, + val phrases: List, + val excludedTerms: List, + val excludedPhrases: List, + val includeSites: List, + val excludeSites: List, + val inTitle: List, + val notInTitle: List, + val inUrl: List, + val notInUrl: List, + val fileTypes: List, + val afterMillis: Long?, + val beforeMillis: Long?, +) { + /** True when any operator is enforced locally by [matches] (everything but plain terms/phrases/OR). */ + val hasFilters: Boolean + get() = + excludedTerms.isNotEmpty() || excludedPhrases.isNotEmpty() || + includeSites.isNotEmpty() || excludeSites.isNotEmpty() || + inTitle.isNotEmpty() || notInTitle.isNotEmpty() || + inUrl.isNotEmpty() || notInUrl.isNotEmpty() || + fileTypes.isNotEmpty() || afterMillis != null || beforeMillis != null + + /** + * True when an aggregated result survives every locally-enforced operator in this query. + * + * Positive [phrases] and plain terms are deliberately NOT checked here: they were already sent + * upstream in [engineQuery] and drive lexical relevance ([org.searchmob.engine.aggregate.Relevance]). + * A snippet is a partial excerpt of the page, so demanding the phrase appear in the title/snippet + * would reject results whose full body matches but whose short excerpt happens not to quote it; + * the engines and the relevance ranker are trusted to have already done that job. + * + * A date-window bound ([afterMillis]/[beforeMillis]) excludes a result with no known + * [publishedMillis], deliberately: the user explicitly asked for a window, so an undated result + * cannot be confirmed to be in it and is treated as a miss rather than let through. + */ + fun matches( + title: String, + url: String, + snippet: String, + publishedMillis: Long?, + ): Boolean { + val host = DomainRanker.host(url) + if (includeSites.isNotEmpty() && (host == null || includeSites.none { siteMatches(it, host) })) return false + if (excludeSites.isNotEmpty() && host != null && excludeSites.any { siteMatches(it, host) }) return false + if (inTitle.any { !title.contains(it, ignoreCase = true) }) return false + if (notInTitle.any { title.contains(it, ignoreCase = true) }) return false + val lowerUrl = url.lowercase() + if (inUrl.any { !lowerUrl.contains(it.lowercase()) }) return false + if (notInUrl.any { lowerUrl.contains(it.lowercase()) }) return false + if (fileTypes.isNotEmpty()) { + val extension = extensionOf(url) + if (extension == null || extension !in fileTypes) return false + } + if (afterMillis != null || beforeMillis != null) { + if (publishedMillis == null) return false + if (afterMillis != null && publishedMillis < afterMillis) return false + if (beforeMillis != null && publishedMillis >= beforeMillis) return false + } + if (excludedTerms.isNotEmpty()) { + val words = wordsOf("$title $snippet ${host.orEmpty()}") + if (excludedTerms.any { it.lowercase() in words }) return false + } + val haystack = "$title $snippet" + if (excludedPhrases.any { haystack.contains(it, ignoreCase = true) }) return false + return true + } + + /** + * Whether [entry] (a normalized `site:`/`-site:` value) covers [host]. An entry starting with `.` + * (a bare TLD like `.edu`) matches any host ending in it; otherwise the entry must equal the host or + * be one of its parent domains (`example.com` covers `docs.example.com` but not `notexample.com`). + */ + private fun siteMatches( + entry: String, + host: String, + ): Boolean = if (entry.startsWith(".")) host.endsWith(entry) else host == entry || host.endsWith(".$entry") + + /** + * The lowercased extension of [url]'s last path segment (query string and fragment ignored), or + * null when the path has no segment or that segment has no dot. Parsed via [URI] rather than naive + * string-splitting so a bare `https://example.com` never misreads its TLD (the `.com` in the host) + * as a file extension. + */ + private fun extensionOf(url: String): String? { + val path = runCatching { URI(url.trim()).path }.getOrNull() ?: return null + val lastSegment = path.substringAfterLast('/') + if (!lastSegment.contains('.')) return null + return lastSegment.substringAfterLast('.').lowercase().takeIf { it.isNotEmpty() } + } + + /** Lowercased maximal letter/digit runs in [text], for whole-word exclusion matching. */ + private fun wordsOf(text: String): Set { + val out = HashSet() + val current = StringBuilder() + for (ch in text) { + if (ch.isLetterOrDigit()) { + current.append(ch.lowercaseChar()) + } else if (current.isNotEmpty()) { + out.add(current.toString()) + current.setLength(0) + } + } + if (current.isNotEmpty()) out.add(current.toString()) + return out + } +} + +/** + * Parses Google-fu style search operators (`site:`, `-exclude`, `"exact phrase"`, `intitle:`, `inurl:`, + * `filetype:`/`ext:`, `before:`/`after:`, `OR`/`|`) out of a raw query into a [ParsedQuery]. + * + * Pure and deterministic: no I/O, no locale/clock dependence beyond UTC date math, so the same input + * always parses the same way. The server truncates queries at 512 characters before they ever reach + * here, so this has to tolerate garbage (an unterminated quote, a bare `-`, an empty `site:`) without + * throwing, never silently dropping a token it does not understand — anything it cannot make sense of + * falls back to being treated as ordinary query text. + */ +object QueryOperators { + private val RECOGNIZED_OPS = setOf("site", "intitle", "inurl", "filetype", "ext", "before", "after") + private val YEAR = Regex("^(\\d{4})$") + private val YEAR_MONTH = Regex("^(\\d{4})-(\\d{2})$") + private val FULL_DASH = Regex("^(\\d{4})-(\\d{2})-(\\d{2})$") + private val FULL_SLASH = Regex("^(\\d{4})/(\\d{2})/(\\d{2})$") + + fun parse(raw: String): ParsedQuery { + val acc = Accumulator() + for (token in tokenize(raw)) { + if (token == "OR" || token == "|") { + acc.engineParts.add(token) + continue + } + + val negated = token.length > 1 && token.startsWith("-") + val body = if (negated) token.substring(1) else token + + if (body.startsWith("\"")) { + // A blank phrase (a stray `"` or `-"`) is dropped entirely: an empty excluded phrase + // would match every result (`contains("")` is always true) and filter everything out. + val phrase = unquote(body) + if (phrase.isBlank()) continue + if (negated) { + acc.excludedPhrases.add(phrase) + acc.engineParts.add("-\"$phrase\"") + } else { + acc.phrases.add(phrase) + acc.cleanParts.add(phrase) + acc.engineParts.add("\"$phrase\"") + } + continue + } + + val colonIndex = body.indexOf(':') + val opName = if (colonIndex > 0) body.substring(0, colonIndex).lowercase() else null + if (opName != null && opName in RECOGNIZED_OPS) { + val valueRaw = body.substring(colonIndex + 1) + if (applyOperator(opName, negated, token, valueRaw, acc)) continue + } + + if (negated) { + acc.excludedTerms.add(body) + acc.engineParts.add(token) + } else { + acc.cleanParts.add(token) + acc.engineParts.add(token) + } + } + + return ParsedQuery( + raw = raw, + cleanText = acc.cleanParts.joinToString(" "), + engineQuery = acc.engineParts.joinToString(" "), + phrases = acc.phrases, + excludedTerms = acc.excludedTerms, + excludedPhrases = acc.excludedPhrases, + includeSites = acc.includeSites, + excludeSites = acc.excludeSites, + inTitle = acc.inTitle, + notInTitle = acc.notInTitle, + inUrl = acc.inUrl, + notInUrl = acc.notInUrl, + fileTypes = acc.fileTypes, + afterMillis = acc.afterMillis, + beforeMillis = acc.beforeMillis, + ) + } + + /** The mutable accumulators [parse] fills in as it walks the tokens, in original token order. */ + private class Accumulator { + val cleanParts = ArrayList() + val engineParts = ArrayList() + val phrases = ArrayList() + val excludedTerms = ArrayList() + val excludedPhrases = ArrayList() + val includeSites = ArrayList() + val excludeSites = ArrayList() + val inTitle = ArrayList() + val notInTitle = ArrayList() + val inUrl = ArrayList() + val notInUrl = ArrayList() + val fileTypes = ArrayList() + var afterMillis: Long? = null + var beforeMillis: Long? = null + } + + /** + * Handles one `op:value` token, mutating [acc] for the recognized operators. + * + * Returns false to signal that the token was NOT actually consumed as an operator (a `-` negated + * `filetype:`/`ext:`/`before:`/`after:` has no defined negated meaning), so the caller falls back to + * treating it as a plain `-word` exclusion. + */ + private fun applyOperator( + opName: String, + negated: Boolean, + token: String, + valueRaw: String, + acc: Accumulator, + ): Boolean { + when (opName) { + "site" -> { + val value = normalizeSite(unquoteOrRaw(valueRaw)) + if (value.isEmpty()) return true // an empty operator is dropped entirely + if (negated) acc.excludeSites.add(value) else acc.includeSites.add(value) + acc.engineParts.add(if (negated) "-site:$value" else "site:$value") + } + "filetype", "ext" -> { + if (negated) return false // no defined negated meaning; caller treats it as -word + val value = normalizeFileType(unquoteOrRaw(valueRaw)) + if (value.isEmpty()) return true + acc.fileTypes.add(value) + acc.engineParts.add("filetype:$value") + } + "intitle", "inurl" -> { + val value = unquoteOrRaw(valueRaw) + if (value.isEmpty()) return true + if (negated) { + if (opName == "intitle") acc.notInTitle.add(value) else acc.notInUrl.add(value) + // -intitle:/-inurl: are locally-enforced only; dropped from the upstream query. + } else { + if (opName == "intitle") acc.inTitle.add(value) else acc.inUrl.add(value) + acc.cleanParts.add(value) + acc.engineParts.add(value) + } + } + "before", "after" -> { + if (negated) return false // no defined negated meaning; caller treats it as -word + val value = unquoteOrRaw(valueRaw) + if (value.isEmpty()) return true + val millis = parseDate(value) + if (millis == null) { + // Never silently drop something we could not parse: keep the whole token as text. + acc.cleanParts.add(token) + acc.engineParts.add(token) + } else { + if (opName == "after") acc.afterMillis = millis else acc.beforeMillis = millis + // before:/after: are locally-enforced only; dropped from the upstream query. + } + } + } + return true + } + + /** Split [raw] on whitespace, except inside `"..."`; an unterminated quote runs to end of string. */ + private fun tokenize(raw: String): List { + val tokens = ArrayList() + val current = StringBuilder() + var inQuotes = false + for (ch in raw) { + when { + ch == '"' -> { + inQuotes = !inQuotes + current.append(ch) + } + ch.isWhitespace() && !inQuotes -> { + if (current.isNotEmpty()) { + tokens.add(current.toString()) + current.setLength(0) + } + } + else -> current.append(ch) + } + } + if (current.isNotEmpty()) tokens.add(current.toString()) + return tokens + } + + /** Strip the surrounding quotes off [text] (which starts with `"`); an unterminated one runs to the end. */ + private fun unquote(text: String): String { + val closing = text.indexOf('"', 1) + return if (closing >= 0) text.substring(1, closing) else text.substring(1) + } + + private fun unquoteOrRaw(text: String): String = if (text.startsWith("\"")) unquote(text) else text + + /** `*.example.com` / `example.com.` -> `example.com`. */ + private fun normalizeSite(value: String): String { + var v = value.trim() + if (v.startsWith("*.")) v = v.substring(2) + return v.trimEnd('.').lowercase() + } + + /** `.PDF` / `PDF` -> `pdf`. */ + private fun normalizeFileType(value: String): String = value.trim().lowercase().removePrefix(".") + + /** + * `YYYY`, `YYYY-MM`, `YYYY-MM-DD`, or `YYYY/MM/DD` -> UTC epoch millis at the start of that day + * (year/month default to day 1 / January 1). Null for anything that does not match one of those + * shapes or names an impossible calendar date (e.g. month 13). + */ + private fun parseDate(value: String): Long? { + val date = + when { + YEAR.matches(value) -> runCatching { LocalDate.of(value.toInt(), 1, 1) }.getOrNull() + YEAR_MONTH.matches(value) -> { + val (y, m) = YEAR_MONTH.find(value)!!.destructured + runCatching { LocalDate.of(y.toInt(), m.toInt(), 1) }.getOrNull() + } + FULL_DASH.matches(value) -> { + val (y, m, d) = FULL_DASH.find(value)!!.destructured + runCatching { LocalDate.of(y.toInt(), m.toInt(), d.toInt()) }.getOrNull() + } + FULL_SLASH.matches(value) -> { + val (y, m, d) = FULL_SLASH.find(value)!!.destructured + runCatching { LocalDate.of(y.toInt(), m.toInt(), d.toInt()) }.getOrNull() + } + else -> null + } + return date?.atStartOfDay(ZoneOffset.UTC)?.toInstant()?.toEpochMilli() + } +} diff --git a/app/src/main/java/org/searchmob/server/SearchServer.kt b/app/src/main/java/org/searchmob/server/SearchServer.kt index f3f79f0..e8eb15e 100644 --- a/app/src/main/java/org/searchmob/server/SearchServer.kt +++ b/app/src/main/java/org/searchmob/server/SearchServer.kt @@ -105,6 +105,7 @@ import java.net.InetSocketAddress import java.net.ServerSocket import java.net.URI import java.net.URLEncoder +import java.security.MessageDigest import java.security.SecureRandom import java.util.Base64 @@ -214,17 +215,38 @@ fun Application.searchModule( // Run before routing on every request: conservative security headers always, plus the Host // allowlist and the token gate for non-loopback clients. intercept(ApplicationCallPipeline.Plugins) { - call.response.headers.append("Referrer-Policy", "no-referrer", safeOnly = false) + // `same-origin`, not `no-referrer`: a same-origin request (including our OWN served forms) + // still carries its Referer/Origin to us, which `sameOrigin()` needs to tell a genuine + // same-origin POST apart from a cross-site one that forces an opaque `Origin: null` (see + // `sameOrigin()`'s doc comment for the attack `no-referrer` used to enable). A cross-origin + // navigation away from us still sends nothing, same as before, and every result/summary/action + // link additionally carries `rel="noreferrer"` as a second, redundant layer. + call.response.headers.append("Referrer-Policy", "same-origin", safeOnly = false) + call.response.headers.append("Content-Security-Policy", CSP, safeOnly = false) call.response.headers.append("X-Content-Type-Options", "nosniff", safeOnly = false) call.response.headers.append("X-Frame-Options", "DENY", safeOnly = false) + // Queries, titles, snippets, and Settings values must never persist in a browser or + // intermediary cache once the page is gone. + call.response.headers.append("Cache-Control", "no-store", safeOnly = false) + // We never touch the camera/location/microphone; deny every embedder (and ourselves) the asks. + call.response.headers.append( + "Permissions-Policy", + "camera=(), geolocation=(), microphone=()", + safeOnly = false, + ) if (!isLoopbackHost(call.request.origin.remoteHost)) { val host = hostnameOnly(call.request.headers["Host"].orEmpty()) if (host.isNotEmpty() && !hostHeaderAllowed(host, allowedHosts)) { call.respondText("Bad Request: host not allowed", status = HttpStatusCode.BadRequest) return@intercept finish() } - val tokenOk = !accessToken.isNullOrEmpty() && call.request.queryParameters["token"] == accessToken - if (call.request.path() in GATED_PATHS && !tokenOk) { + val presented = + presentedToken( + call.request.queryParameters["token"], + call.request.headers["Authorization"], + call.request.headers["X-SearchMob-Token"], + ) + if (call.request.path() in GATED_PATHS && !tokenMatches(presented, accessToken)) { call.respondText("Forbidden", status = HttpStatusCode.Forbidden) return@intercept finish() } @@ -630,6 +652,60 @@ internal fun isLoopbackHost(host: String): Boolean { /** Query routes gated by the access token for non-loopback clients in network mode. */ private val GATED_PATHS = setOf("/search", "/api/search", "/suggest") +/** + * Content-Security-Policy sent on every response. `kotlinx.html` HTML-escapes every piece of dynamic + * content it renders (query text, titles, snippets, domains, settings values, ...), so the small + * inline theme/reveal `