diff --git a/NAMESPACE b/NAMESPACE index 877b309..6754fcc 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -42,6 +42,7 @@ export(lnk_pipeline_setup) export(lnk_pipeline_species) export(lnk_points_snap) export(lnk_presence) +export(lnk_rollup_wsg) export(lnk_rules_build) export(lnk_score) export(lnk_source) diff --git a/R/lnk_compare_rollup.R b/R/lnk_compare_rollup.R index d6f72b4..a281297 100644 --- a/R/lnk_compare_rollup.R +++ b/R/lnk_compare_rollup.R @@ -41,10 +41,13 @@ #' (e.g. `c("BT","CO")`) to restrict the rollup to. Default `NULL` #' discovers the set from PG. #' -#' @return A tibble with one row per (species, habitat_type) — 7 -#' habitat types per species. Columns: `wsg`, `species`, +#' @return A tibble with one row per (species, habitat_type) — 8 +#' habitat types per species (the 7 habitat km/ha types plus +#' `accessible` km, link#221). Columns: `wsg`, `species`, #' `habitat_type`, `unit` (`km` | `ha`), `link_value`, `ref_value`, -#' `diff_pct`. +#' `diff_pct`. `accessible`'s `ref_value` is sourced tunnel-free from +#' `fresh.streams_vw_bcfp` for the salmon group (CH/CM/CO/PK/SK); other +#' species carry `NA` until their reference path lands (link#221 Phase 3). #' #' @examples #' \dontrun{ @@ -120,7 +123,7 @@ lnk_compare_rollup <- function(conn, aoi, cfg, conn = conn, cfg = cfg, aoi = aoi, species = species) rollup_ref <- .lnk_compare_wsg_rollup_reference( # nolint reference = reference, conn_ref = conn_ref, - aoi = aoi, species = species) + aoi = aoi, species = species, conn = conn) .lnk_compare_wsg_assemble_rollup( # nolint aoi = aoi, species = species, @@ -195,35 +198,43 @@ lnk_compare_rollup <- function(conn, aoi, cfg, et_lake_sql <- "(1500, 1525)" et_wetland_sql <- "(1700)" - union_streams <- paste(vapply(species, function(sp) { - sp_lit <- DBI::dbQuoteLiteral(conn, sp) - sprintf( - "SELECT %s AS species_code, s.id_segment, s.length_metre, - s.edge_type, h.spawning, h.rearing - FROM %s.streams s - JOIN %s.streams_habitat_%s h ON s.id_segment = h.id_segment AND s.watershed_group_code = h.watershed_group_code - WHERE s.watershed_group_code = %s", - sp_lit, tn$schema, tn$schema, tolower(sp), aoi_lit) # nolint: indentation_linter - }, character(1)), collapse = "\n UNION ALL\n ") + # Habitat km sums delegate to the single predicate-driven roll-up + # primitive (link#221) so there is one per-(WSG, species) km query + # builder. lnk_rollup_wsg exposes `length_metre` / `edge_type` / + # `access` / `spawning` / `rearing` under generic aliases; the first + # five metrics reproduce the historical shape, and `accessible_km` + # (link#221) sums link's per-species access model + # (`streams_access.access_ IN (1,2)`, LEFT-joined by + # lnk_rollup_wsg). It returns `wsg` + `species` + metrics — drop + # `wsg` and rename `species` -> `species_code` to preserve this + # helper's `list(km, lake_ha, wetland_ha)` contract. + # COALESCE(..., 0): an empty FILTER set sums to NULL, but the + # historical CASE-WHEN-ELSE-0 form returned a measured 0 for a + # species present in the WSG with no segments in that slice. Preserve + # that 0 (not NA) so downstream parity diff_pct is unchanged. + km_metrics <- c( + spawning_km = + "round(COALESCE(sum(length_metre) FILTER (WHERE spawning), 0)::numeric / 1000, 2)", # nolint: line_length_linter + rearing_km = + "round(COALESCE(sum(length_metre) FILTER (WHERE rearing), 0)::numeric / 1000, 2)", # nolint: line_length_linter + rearing_stream_km = sprintf( + "round(COALESCE(sum(length_metre) FILTER (WHERE rearing AND edge_type IN %s), 0)::numeric / 1000, 2)", # nolint: line_length_linter + et_stream_sql), + rearing_lake_centerline_km = sprintf( + "round(COALESCE(sum(length_metre) FILTER (WHERE rearing AND edge_type IN %s), 0)::numeric / 1000, 2)", # nolint: line_length_linter + et_lake_sql), + rearing_wetland_centerline_km = sprintf( + "round(COALESCE(sum(length_metre) FILTER (WHERE rearing AND edge_type IN %s), 0)::numeric / 1000, 2)", # nolint: line_length_linter + et_wetland_sql), + accessible_km = + "round(COALESCE(sum(length_metre) FILTER (WHERE access IN (1, 2)), 0)::numeric / 1000, 2)") # nolint: line_length_linter - km <- DBI::dbGetQuery(conn, sprintf(" - SELECT species_code, - round(SUM(CASE WHEN spawning THEN length_metre ELSE 0 END)::numeric - / 1000, 2) AS spawning_km, - round(SUM(CASE WHEN rearing THEN length_metre ELSE 0 END)::numeric - / 1000, 2) AS rearing_km, - round(SUM(CASE WHEN rearing AND edge_type IN %s - THEN length_metre ELSE 0 END)::numeric / 1000, 2) - AS rearing_stream_km, - round(SUM(CASE WHEN rearing AND edge_type IN %s - THEN length_metre ELSE 0 END)::numeric / 1000, 2) - AS rearing_lake_centerline_km, - round(SUM(CASE WHEN rearing AND edge_type IN %s - THEN length_metre ELSE 0 END)::numeric / 1000, 2) - AS rearing_wetland_centerline_km - FROM (%s) per_species - GROUP BY species_code ORDER BY species_code", - et_stream_sql, et_lake_sql, et_wetland_sql, union_streams)) # nolint: indentation_linter + km <- lnk_rollup_wsg( # nolint: object_usage_linter + conn = conn, aoi = aoi, species = species, + schema = tn$schema, metrics = km_metrics) + km$wsg <- NULL + names(km)[names(km) == "species"] <- "species_code" + km <- km[, c("species_code", names(km_metrics))] # Lake / wetland ha — DISTINCT waterbody_key joins to fwa polygon # tables avoid double-counting multi-segment lakes/wetlands. diff --git a/R/lnk_compare_wsg.R b/R/lnk_compare_wsg.R index d091835..9099cf7 100644 --- a/R/lnk_compare_wsg.R +++ b/R/lnk_compare_wsg.R @@ -51,12 +51,13 @@ #' deprecation warning when supplied. #' #' @return A list with two elements: -#' - `rollup`: tibble with one row per (species, habitat_type) — 7 +#' - `rollup`: tibble with one row per (species, habitat_type) — 8 #' habitat types: `spawning`, `rearing`, `lake_rearing`, #' `wetland_rearing`, `rearing_stream`, `rearing_lake_centerline`, -#' `rearing_wetland_centerline`. Columns: `wsg`, `species`, -#' `habitat_type`, `unit` (`km` | `ha`), `link_value`, -#' `ref_value`, `diff_pct`. +#' `rearing_wetland_centerline`, `accessible` (km, link#221). +#' Columns: `wsg`, `species`, `habitat_type`, `unit` (`km` | `ha`), +#' `link_value`, `ref_value`, `diff_pct`. `accessible`'s `ref_value` +#' is `NA` until the tunnel-free reference path lands. #' - `mapping_code`: tibble with one row per species — segment-level #' match stats vs `bcfishpass.streams_mapping_code`. Columns: #' `wsg`, `species`, `total_segs`, `match_pct`, `n_diffs`, @@ -311,21 +312,63 @@ lnk_compare_wsg <- function(conn, aoi, cfg, loaded, #' Compute reference-side rollup queries for the requested `reference` #' #' Dispatches on `reference`. Currently only `"bcfishpass"` is wired — -#' queries `bcfishpass.habitat_linear_` per species, joined to the -#' same `fwa_lakes_poly` / `fwa_wetlands_poly` for the ha columns -#' bcfp doesn't materialize natively. +#' queries `bcfishpass.habitat_linear_` per species (on `conn_ref`, +#' the tunnel), joined to the same `fwa_lakes_poly` / `fwa_wetlands_poly` +#' for the ha columns bcfp doesn't materialize natively. The `accessible_km` +#' reference is sourced separately + tunnel-free from `fresh.streams_vw_bcfp` +#' on the local `conn` (see [.lnk_compare_wsg_accessible_ref()]); the two +#' reference sources stay intentionally decoupled (link#221). #' #' @noRd .lnk_compare_wsg_rollup_reference <- function(reference, conn_ref, - aoi, species) { + aoi, species, conn) { if (reference == "bcfishpass") { - return(.lnk_compare_wsg_rollup_bcfishpass( - conn_ref = conn_ref, aoi = aoi, species = species)) + ref <- .lnk_compare_wsg_rollup_bcfishpass( + conn_ref = conn_ref, aoi = aoi, species = species) # nolint: indentation_linter + ref$accessible_km <- .lnk_compare_wsg_accessible_ref( + conn = conn, aoi = aoi, species = ref$species_code) # nolint: indentation_linter + return(ref) } stop("Unknown reference '", reference, "' in dispatch.", call. = FALSE) } +#' Tunnel-free `accessible_km` reference from `fresh.streams_vw_bcfp` +#' +#' Sums `length_metre` where the species' bcfp barrier-group column is +#' empty (`= ''` = no barrier downstream = accessible). Sourced from the +#' local province-wide snapshot (`conn`), NOT the tunnel — the `:63333` +#' tunnel is dead on M1 and the snapshot is authoritative. Only the +#' Phase-1-proven salmon group (CH/CM/CO/PK/SK → +#' `barriers_ch_cm_co_pk_sk_dnstr`) is wired; other species short-circuit +#' to `NA` (their reference lands in a later phase, each needing its own +#' proof). `barrier_group` values are a trusted constant whitelist — safe +#' to interpolate. +#' +#' @noRd +.lnk_compare_wsg_accessible_ref <- function(conn, aoi, species) { + barrier_group <- c( + CH = "barriers_ch_cm_co_pk_sk_dnstr", + CM = "barriers_ch_cm_co_pk_sk_dnstr", + CO = "barriers_ch_cm_co_pk_sk_dnstr", + PK = "barriers_ch_cm_co_pk_sk_dnstr", + SK = "barriers_ch_cm_co_pk_sk_dnstr" + ) + aoi_lit <- DBI::dbQuoteLiteral(conn, aoi) + vapply(species, function(sp) { + grp <- barrier_group[toupper(sp)] + if (is.na(grp)) return(NA_real_) + res <- DBI::dbGetQuery(conn, sprintf(" + SELECT round((COALESCE(sum(length_metre), 0) / 1000)::numeric, 2) + AS accessible_km + FROM fresh.streams_vw_bcfp + WHERE watershed_group_code = %s AND %s = ''", + aoi_lit, unname(grp))) # nolint: indentation_linter + res$accessible_km[1] + }, numeric(1), USE.NAMES = FALSE) +} + + #' Reference dispatch: bcfishpass tunnel #' #' Mirrors the link-side methodology applied to `bcfishpass.habitat_linear_` @@ -434,25 +477,28 @@ lnk_compare_wsg <- function(conn, aoi, cfg, loaded, #' Assemble long-format output tibble from link + ref rollup data #' -#' Produces 7 rows per species (spawning, rearing, lake_rearing, +#' Produces 8 rows per species (spawning, rearing, lake_rearing, #' wetland_rearing, rearing_stream, rearing_lake_centerline, -#' rearing_wetland_centerline). `diff_pct = NA` when `ref_value` is -#' `NA` (species not modelled by reference) or `0` (avoid div-by-zero -#' even when the measurement is real). +#' rearing_wetland_centerline, accessible). `diff_pct = NA` when +#' `ref_value` is `NA` (species not modelled by reference, or — for +#' `accessible` — the tunnel-free reference path not yet wired) or `0` +#' (avoid div-by-zero even when the measurement is real). #' #' @noRd .lnk_compare_wsg_assemble_rollup <- function(aoi, species, rollup_link, rollup_ref) { habitat_types <- c( "spawning", "rearing", "lake_rearing", "wetland_rearing", - "rearing_stream", "rearing_lake_centerline", "rearing_wetland_centerline" + "rearing_stream", "rearing_lake_centerline", "rearing_wetland_centerline", + "accessible" ) units <- c( spawning = "km", rearing = "km", lake_rearing = "ha", wetland_rearing = "ha", rearing_stream = "km", rearing_lake_centerline = "km", - rearing_wetland_centerline = "km" + rearing_wetland_centerline = "km", + accessible = "km" ) col_suffix <- c( spawning = "spawning_km", rearing = "rearing_km", @@ -460,7 +506,8 @@ lnk_compare_wsg <- function(conn, aoi, cfg, loaded, wetland_rearing = "wetland_rearing_ha", rearing_stream = "rearing_stream_km", rearing_lake_centerline = "rearing_lake_centerline_km", - rearing_wetland_centerline = "rearing_wetland_centerline_km" + rearing_wetland_centerline = "rearing_wetland_centerline_km", + accessible = "accessible_km" ) sp_col <- rep(species, each = length(habitat_types)) @@ -483,7 +530,8 @@ lnk_compare_wsg <- function(conn, aoi, cfg, loaded, wetland_rearing = rollup_link$wetland_ha, rearing_stream = rollup_link$km, rearing_lake_centerline = rollup_link$km, - rearing_wetland_centerline = rollup_link$km + rearing_wetland_centerline = rollup_link$km, + accessible = rollup_link$km ) for (i in seq_len(nrow(out))) { diff --git a/R/lnk_pipeline_break.R b/R/lnk_pipeline_break.R index 2440e49..4273b1e 100644 --- a/R/lnk_pipeline_break.R +++ b/R/lnk_pipeline_break.R @@ -21,7 +21,7 @@ #' | name | source table | role | classify label | #' |---|---|---|---| #' | `observations` | `.observations_breaks` | fish observations from `bcfishobs.observations`, WSG- and species-filtered, exclusions applied | (informational; not a barrier) | -#' | `gradient_minimal` | `.gradient_barriers_minimal` | minimal-reduced gradient barriers (per-model 15/20/25/30%) | classify uses the FULL set with `gradient_` labels | +#' | `gradient_minimal` | `.gradient_barriers_minimal` | FULL per-model gradient barriers (15/20/25/30%) + falls — every position breaks the network so access gates at each frontier (#223); NOT minimal-reduced despite the legacy name | classify uses the same FULL set with `gradient_` labels | #' | `falls` | `.falls` | natural waterfalls from `whse_basemapping.fwa_obstacles_sp` (loaded by `prep_load_aux`); each fall is its own barrier (NOT minimal-reduced) | `blocked` | #' | `barriers_definite` | `.barriers_definite` | `user_barriers_definite.csv` for the AOI | `blocked` | #' | `subsurfaceflow` | `.barriers_subsurfaceflow` | FWA `edge_type IN (1410, 1425)` start points; honours `user_barriers_definite_control`. Opt-in (only built when listed) | `blocked` | diff --git a/R/lnk_pipeline_prepare.R b/R/lnk_pipeline_prepare.R index 968afc4..104f3c3 100644 --- a/R/lnk_pipeline_prepare.R +++ b/R/lnk_pipeline_prepare.R @@ -15,9 +15,10 @@ #' `lnk_barrier_overrides()` to compute the per-species skip list. #' User-definite barriers are intentionally excluded here and #' consumed by later phases directly — bcfishpass parity. -#' - Per-model barrier tables reduced to the minimal downstream-most -#' set via [fresh::frs_barriers_minimal()], then unioned into -#' `gradient_barriers_minimal` for segmentation +#' - Per-model barrier tables (gradient class-filtered + falls) unioned +#' into `gradient_barriers_minimal` — the FULL per-model position set +#' (NOT minimal-reduced, despite the legacy name), so streams break at +#' every frontier and per-segment access gating is correct (#223) #' - Base stream segments (`fresh.streams`) loaded from FWA with #' channel width, stream order parent, GENERATED gradient / measures #' / length columns, and a unique `id_segment` @@ -31,9 +32,8 @@ #' - `.barriers_subsurfaceflow` (only when subsurfaceflow opted in #' via `cfg$pipeline$break_order`) #' - `.barrier_overrides` -#' - `.barriers_` + `.barriers__min` -#' per-model pre/post minimal reduction -#' - `.gradient_barriers_minimal` (union of minimal positions) +#' - `.barriers_` — per-model gradient + falls positions +#' - `.gradient_barriers_minimal` (union of per-model raw positions) #' - `fresh.streams` (base segments — not namespaced by AOI; fresh #' owns its output schema) #' - `.dams` (only when `conn_tunnel` is supplied) — pulled @@ -567,9 +567,8 @@ lnk_pipeline_prepare <- function(conn, aoi, cfg, loaded, schema, for (model_name in names(models)) { class_filter <- paste(models[[model_name]], collapse = ", ") model_tbl <- paste0(schema, ".barriers_", model_name) - min_tbl <- paste0(model_tbl, "_min") - # Build pre-minimal set: gradient (class-filtered) + falls + # Build the per-model break set: gradient (class-filtered) + falls. .lnk_db_execute(conn, sprintf("DROP TABLE IF EXISTS %s", model_tbl)) .lnk_db_execute(conn, sprintf( "CREATE TABLE %s AS @@ -589,8 +588,17 @@ lnk_pipeline_prepare <- function(conn, aoi, cfg, loaded, schema, model_tbl, schema, class_filter, schema, .lnk_quote_literal(aoi))) - fresh::frs_barriers_minimal(conn, from = model_tbl, to = min_tbl) - minimal_tbls <- c(minimal_tbls, min_tbl) + # Feed the RAW per-model positions into the break network — every gradient / + # falls barrier must split the streams so the per-segment downstream-barrier + # access check (lnk_pipeline_access) can gate the reach ABOVE each frontier. + # Do NOT frs_barriers_minimal() here: minimal reduction keeps only the + # downstream-most position per flow path (correct for an access DECISION, + # wrong as a SEGMENTATION source — it strips the interior frontier so a single + # segment straddles the barrier and its whole reach is mislabelled accessible; + # #223). This mirrors the orphan-set treatment below and matches bcfishpass, + # which segments at every gradient barrier. Table name kept as + # gradient_barriers_minimal for now; a follow-up issue renames it. + minimal_tbls <- c(minimal_tbls, model_tbl) } # Orphan classes: thresholds in `classes` that fall below every species's diff --git a/R/lnk_rollup_wsg.R b/R/lnk_rollup_wsg.R new file mode 100644 index 0000000..9e9b9ac --- /dev/null +++ b/R/lnk_rollup_wsg.R @@ -0,0 +1,160 @@ +#' Roll up per-(WSG, species) length metrics from persisted state +#' +#' Reusable, predicate-driven roll-up over link's persisted per-species +#' tables. For each species it joins `.streams` (length + edge +#' type) to `.streams_habitat_` (spawning / rearing flags) +#' and **left-joins** `.streams_access` (per-species +#' `access_` code) on the full PK `(id_segment, +#' watershed_group_code)` (#203), exposes the three species-varying +#' inputs under **generic aliases** — `access` (int: -9 absent / 0 +#' blocked / 1 modelled / 2 observed), `spawning`, `rearing` (bool) — +#' then aggregates by `(watershed_group_code, species_code)`. +#' +#' The `streams_access` join is a LEFT join: access is optional metadata +#' for a length roll-up. When a segment has no `streams_access` row +#' (access not yet built for the WSG), `access` is `NULL` and +#' `accessible_km` resolves to 0 — build it via +#' `lnk_pipeline_run(mapping_code = TRUE)` (or the unconditional access +#' phase). Length is never dropped by a missing access row, so the +#' habitat metrics (`spawning_km`, `rearing_km`) are unaffected. +#' +#' Because the per-species columns are aliased to fixed names, the +#' `metrics` SQL is written **once**, species-agnostic — mirroring +#' `fresh::frs_aggregate()`'s `metrics` / `where` shape. Adding a species +#' is a `species` vector edit, not a query edit. +#' +#' This is a **flat per-WSG `GROUP BY`** — it sums whole-WSG length by +#' `(watershed_group_code, species_code)`. It is distinct from +#' [lnk_aggregate()] / [fresh::frs_aggregate()], which roll habitat up the +#' network *upstream of individual crossings* (point-based traversal). Use +#' this for WSG totals; use those for per-crossing upstream summaries. +#' +#' `accessible_km` sums `access IN (1, 2)` — link's per-species access +#' model on `streams_access`, the number validated against the +#' tunnel-free bcfp reference in `data-raw/parity_crosssection.R` +#' (accessible + spawning + rearing, 8 species x 11 WSGs). It +#' deliberately does **not** use the +#' `accessible` boolean on `streams_habitat_`, which carries +#' different (pre-gating) semantics and diverges from the access model +#' (MORR coho: 3424 km vs the validated 3330 km). +#' +#' @param conn A [DBI::DBIConnection-class] object (from [lnk_db_conn()]). +#' @param aoi Watershed group code (e.g. `"MORR"`). Uppercase 3-5 letters. +#' @param species Character vector of species codes (e.g. `c("CO","BT")`). +#' Each must name existing `.streams_habitat_` and +#' `.streams_access.access_`. Restricted to alpha +#' characters — interpolated into identifiers, so validated to make +#' SQL injection structurally impossible. +#' @param schema Persist schema holding `streams`, `streams_access`, +#' `streams_habitat_`. Default `"fresh"`. Validated against the SQL +#' identifier whitelist. +#' @param metrics Named character vector: names are output columns, +#' values are SQL aggregate expressions over the generic aliases +#' `length_metre`, `access`, `spawning`, `rearing`. Default emits +#' `accessible_km`, `spawning_km`, `rearing_km`. Raw SQL — trusted +#' caller input, like `frs_aggregate()`. +#' @param where Character or `NULL`. Optional SQL predicate applied to the +#' per-species rows before aggregation (aliases available). Default +#' `NULL`. +#' +#' @return A data.frame with one row per `(wsg, species)` and one column +#' per metric. Columns: `wsg`, `species`, then `names(metrics)`. +#' +#' @examples +#' \dontrun{ +#' conn <- lnk_db_conn() +#' # Coho accessible / spawning / rearing km for Morice, from persisted state. +#' lnk_rollup_wsg(conn, aoi = "MORR", species = "CO") +#' +#' # Custom metric: count accessible segments per species. +#' lnk_rollup_wsg(conn, aoi = "MORR", species = c("CO", "BT"), +#' metrics = c(n_accessible = "COUNT(*) FILTER (WHERE access IN (1, 2))")) +#' } +#' +#' @family compare +#' @seealso [lnk_compare_rollup()], [lnk_aggregate()], +#' [fresh::frs_aggregate()] +#' @export +# nolint start: indentation_linter +lnk_rollup_wsg <- function(conn, aoi, species, + schema = "fresh", + metrics = c( + accessible_km = + "round(sum(length_metre) FILTER (WHERE access IN (1, 2))::numeric / 1000, 2)", # nolint: line_length_linter + spawning_km = + "round(sum(length_metre) FILTER (WHERE spawning)::numeric / 1000, 2)", # nolint: line_length_linter + rearing_km = + "round(sum(length_metre) FILTER (WHERE rearing)::numeric / 1000, 2)" # nolint: line_length_linter + ), + where = NULL) { + stopifnot( + inherits(conn, "DBIConnection"), + is.character(aoi), length(aoi) == 1L, nzchar(aoi), + grepl("^[A-Z]{3,5}$", aoi), + is.character(species), length(species) >= 1L, all(nzchar(species)), + # Species suffix is interpolated into `streams_habitat_` and + # `access_`; alpha-only makes injection structurally impossible. + all(grepl("^[A-Za-z]+$", species)), + is.character(schema), length(schema) == 1L, + grepl("^[a-z_][a-z0-9_]*$", schema), + is.character(metrics), length(metrics) >= 1L, + !is.null(names(metrics)), all(nzchar(names(metrics))), + is.null(where) || (is.character(where) && length(where) == 1L) + ) + + sql <- .lnk_rollup_wsg_sql( + conn = conn, aoi = aoi, species = species, + schema = schema, metrics = metrics, where = where) + DBI::dbGetQuery(conn, sql) +} + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + +#' Build the per-(WSG, species) roll-up SQL. +#' +#' Split out from [lnk_rollup_wsg()] so the query text is unit-testable +#' without a live connection (arg validation happens in the exported fn). +#' +#' @noRd +.lnk_rollup_wsg_sql <- function(conn, aoi, species, schema, metrics, where) { + aoi_lit <- DBI::dbQuoteLiteral(conn, aoi) + + per_species <- paste(vapply(species, function(sp) { + sp_lit <- DBI::dbQuoteLiteral(conn, toupper(sp)) + sprintf( + "SELECT %s AS species_code, s.watershed_group_code, + s.length_metre, s.edge_type, + a.access_%s AS access, h.spawning, h.rearing + FROM %s.streams s + JOIN %s.streams_habitat_%s h + ON s.id_segment = h.id_segment + AND s.watershed_group_code = h.watershed_group_code + LEFT JOIN %s.streams_access a + ON s.id_segment = a.id_segment + AND s.watershed_group_code = a.watershed_group_code + WHERE s.watershed_group_code = %s", + sp_lit, tolower(sp), + schema, schema, tolower(sp), schema, aoi_lit) + }, character(1)), collapse = "\n UNION ALL\n ") + + cols_metric <- paste( + sprintf("%s AS %s", unname(metrics), names(metrics)), + collapse = ",\n ") + + where_clause <- if (!is.null(where)) paste("\n WHERE", where) else "" + + sprintf( + "SELECT watershed_group_code AS wsg, + species_code AS species, + %s + FROM ( + %s + ) per_species%s + GROUP BY watershed_group_code, species_code + ORDER BY species_code", + cols_metric, per_species, where_clause) +} +# nolint end: indentation_linter diff --git a/data-raw/parity_crosssection.R b/data-raw/parity_crosssection.R new file mode 100644 index 0000000..f223e8d --- /dev/null +++ b/data-raw/parity_crosssection.R @@ -0,0 +1,165 @@ +#!/usr/bin/env Rscript +# parity_crosssection.R — the single accessible + spawning + rearing parity +# validator/proof for the #221 (accessible_km roll-up) + #223 (access-segmentation +# fix) work. Supersedes the earlier accessible_km_proof_co.R (coho-only) and +# accessible_km_fix_validate.R (fix gate) — both folded in here. +# +# Two jobs, one script: +# 1. PARITY SWEEP — link vs bcfp for accessible/spawning/rearing per (WSG, species). +# link side = lnk_rollup_wsg() (#221): streams x streams_access x streams_habitat_ +# on the full PK, accessible = access_ IN (1,2). +# bcfp side = tunnel-free fresh.streams_vw_bcfp, IN (1,2) predicate (bcfp codes +# access/spawning/rearing 0/1/2/3; a bare `= 1` UNDER-counts). No rearing_cm/_pk +# (chum/pink don't rear in freshwater). +# 2. STRUCTURAL — the #223 mechanism proof on the canonical evidence segment +# (FINA blk 359209845, BT frontier 3834.78): streams break at the frontier, the +# reach above is BT-blocked, the accessible reach tops out at the frontier. +# +# Exits non-zero if any ACCESSIBLE pair breaches tol, any STRUCTURAL check fails, or +# any habitat pair breaches tol that is NOT a known parked departure. Habitat parked +# cases (fresh#190 BULK SK) and #189 residence species (link-absent) are allowlisted. +# +# Prereq: each WSG re-run through lnk_pipeline_run(mapping_code = TRUE) with the fix. +# Usage: [LNK_LOAD=loadall] Rscript data-raw/parity_crosssection.R [WSG ...] + +if (identical(Sys.getenv("LNK_LOAD"), "loadall")) { + suppressPackageStartupMessages(pkgload::load_all(quiet = TRUE)) +} else { + suppressPackageStartupMessages(library(link)) +} +suppressPackageStartupMessages({ + library(DBI) + library(glue) +}) + +conn <- lnk_db_conn(dbname = "fwapg", host = "localhost", port = 5432L, + user = "postgres", password = "postgres") +on.exit(try(DBI::dbDisconnect(conn), silent = TRUE), add = TRUE) + +args <- commandArgs(trailingOnly = TRUE) +wsgs <- if (length(args)) toupper(args) else c("FINA", "PARS", "PCEA", "LKEL") +species <- c("bt", "st", "co", "ch", "cm", "pk", "sk", "wct") +rear_sp <- c("bt", "ch", "co", "sk", "st", "wct") +present_min <- 1.0 +tol_acc <- 1.0 # accessible: the #223 fix should make this tight +tol_hab <- 5.0 # spawn / rear: legitimate methodology wiggle +hab_parked <- c("BULK:SK") # documented parked habitat departures (fresh#190) — not a #223 regression +col_map <- c(accessible = "accessible_km", spawning = "spawning_km", rearing = "rearing_km") +bc_prefix <- c(accessible = "acc_", spawning = "spawn_", rearing = "rear_") + +# canonical #223 evidence segment +frontier_blk <- 359209845L +frontier_m <- 3834.78 +frontier_eps <- 1.0 + +bcfp_rollup <- function(w) { + cols <- unlist(lapply(species, function(sp) { + r <- if (sp %in% rear_sp) { + sprintf(paste0("round((coalesce(sum(length_metre) FILTER (WHERE rearing_%s IN (1,2)),0)", + "/1000)::numeric,2) rear_%s"), sp, sp) + } else { + "NULL::numeric rear_x" + } + sprintf("round((coalesce(sum(length_metre) FILTER (WHERE access_%s IN (1,2)),0)/1000)::numeric,2) acc_%s, + round((coalesce(sum(length_metre) FILTER (WHERE spawning_%s IN (1,2)),0)/1000)::numeric,2) spawn_%s, %s", + sp, sp, sp, sp, r) + })) + DBI::dbGetQuery(conn, glue( + "select {paste(cols, collapse=',')} from fresh.streams_vw_bcfp where watershed_group_code = '{w}'")) +} +num <- function(x) { + x <- suppressWarnings(as.numeric(x[1])) + if (length(x) == 0L || is.na(x)) 0 else x +} + +# ---- 1. parity sweep ------------------------------------------------------ +rows <- list() +absent <- character(0) +for (w in wsgs) { + lk <- lnk_rollup_wsg(conn, aoi = w, species = species, schema = "fresh") + bc <- bcfp_rollup(w) + for (sp in species) { + sp_up <- toupper(sp) + lk_row <- lk[lk$species == sp_up, ] + for (m in names(col_map)) { + if (m == "rearing" && !(sp %in% rear_sp)) next + bcv <- num(bc[[paste0(bc_prefix[m], sp)]]) + lkv <- if (nrow(lk_row)) num(lk_row[[col_map[m]]]) else 0 + if (max(lkv, bcv) < present_min) next + if (lkv < present_min && bcv >= present_min) { + absent <- c(absent, sprintf("%s %s %s (bcfp %.0f, link 0)", w, sp_up, m, bcv)) + next + } + pct <- 100 * (lkv - bcv) / bcv + tol <- if (m == "accessible") tol_acc else tol_hab + parked <- m != "accessible" && paste(w, sp_up, sep = ":") %in% hab_parked + rows[[length(rows) + 1L]] <- data.frame( + wsg = w, sp = sp_up, metric = m, link = round(lkv, 1), bcfp = round(bcv, 1), + diff_pct = round(pct, 2), ok = abs(pct) <= tol, parked = parked) + } + } +} +tab <- do.call(rbind, rows) +cat("\n============ accessible + spawn + rear parity (link vs bcfp, tunnel-free) ============\n") +print(tab[, c("wsg", "sp", "metric", "link", "bcfp", "diff_pct", "ok")], row.names = FALSE) + +cat("\n==== summary by metric (accessible tol", tol_acc, "%, habitat tol", tol_hab, "%) ====\n") +for (m in c("accessible", "spawning", "rearing")) { + sub <- tab[tab$metric == m, ] + fails <- sub[!sub$ok, ] + over <- if (nrow(fails)) { + paste0(" OVER: ", paste(sprintf("%s/%s %+.1f%%%s", fails$wsg, fails$sp, fails$diff_pct, + ifelse(fails$parked, " (parked)", "")), collapse = "; ")) + } else { + "" + } + cat(sprintf(" %-11s: %2d pairs / %d WSGs / %d species, max |diff| %5.2f%%, within-tol %d/%d%s\n", + m, nrow(sub), length(unique(sub$wsg)), length(unique(sub$sp)), + max(abs(sub$diff_pct)), sum(sub$ok), nrow(sub), over)) +} +if (length(absent)) { + cat("\n link-absent (bcfp models species, link does not — #189 residence, excluded):\n") + for (a in absent) cat(" ", a, "\n") +} + +# ---- 2. structural checks: canonical #223 evidence segment ---------------- +struct_fail <- character(0) +if ("FINA" %in% wsgs) { + cat("\n==== structural: FINA blk", frontier_blk, "BT frontier", frontier_m, "====\n") + # no fresh.streams segment straddles the frontier (breaks round to integer 3835, as bcfp) + span <- DBI::dbGetQuery(conn, sprintf( + "SELECT count(*)::int n FROM fresh.streams WHERE blue_line_key=%d AND watershed_group_code='FINA' + AND downstream_route_measure < %f AND upstream_route_measure > %f", + frontier_blk, frontier_m - frontier_eps, frontier_m + frontier_eps))$n + # the segment starting at the frontier is BT-blocked; accessible BT reach tops out there + seg <- DBI::dbGetQuery(conn, sprintf( + "SELECT a.access_bt FROM fresh.streams s JOIN fresh.streams_access a + ON s.id_segment=a.id_segment AND s.watershed_group_code=a.watershed_group_code + WHERE s.blue_line_key=%d AND s.watershed_group_code='FINA' + AND abs(s.downstream_route_measure - %f) < %f", frontier_blk, frontier_m, frontier_eps)) + maxupm <- DBI::dbGetQuery(conn, sprintf( + "SELECT coalesce(max(s.upstream_route_measure),0) m FROM fresh.streams s JOIN fresh.streams_access a + ON s.id_segment=a.id_segment AND s.watershed_group_code=a.watershed_group_code + WHERE s.blue_line_key=%d AND s.watershed_group_code='FINA' AND a.access_bt IN (1,2)", frontier_blk))$m + checks <- c( + "no segment straddles the frontier" = span == 0L, + "segment at frontier is BT-blocked" = nrow(seg) > 0L && isTRUE(all(seg$access_bt == 0L)), + "accessible BT reach tops at frontier" = maxupm <= frontier_m + frontier_eps) + for (nm in names(checks)) { + cat(sprintf(" %s: %s\n", ifelse(checks[[nm]], "pass", "FAIL"), nm)) + if (!checks[[nm]]) struct_fail <- c(struct_fail, nm) + } +} + +# ---- verdict -------------------------------------------------------------- +acc_fail <- tab[tab$metric == "accessible" & !tab$ok, ] +hab_fail <- tab[tab$metric != "accessible" & !tab$ok & !tab$parked, ] +cat(sprintf("\nOVERALL: %d/%d parity pairs within tolerance (%d over: %d accessible, %d habitat non-parked)", + sum(tab$ok), nrow(tab), sum(!tab$ok), nrow(acc_fail), nrow(hab_fail))) +n_fail <- nrow(acc_fail) + nrow(hab_fail) + length(struct_fail) +if (n_fail == 0L) { + cat(" — PASS\n") + quit(status = 0) +} +cat(sprintf(" — %d FAILURE(S)\n", n_fail)) +quit(status = 1) diff --git a/man/lnk_access.Rd b/man/lnk_access.Rd index c4f81a0..0e61c3d 100644 --- a/man/lnk_access.Rd +++ b/man/lnk_access.Rd @@ -128,6 +128,7 @@ Other compare: \code{\link{lnk_compare_rollup}()}, \code{\link{lnk_compare_wsg}()}, \code{\link{lnk_mapping_code}()}, -\code{\link{lnk_parity_annotate}()} +\code{\link{lnk_parity_annotate}()}, +\code{\link{lnk_rollup_wsg}()} } \concept{compare} diff --git a/man/lnk_compare_mapping_code.Rd b/man/lnk_compare_mapping_code.Rd index 7e64f18..fe301e9 100644 --- a/man/lnk_compare_mapping_code.Rd +++ b/man/lnk_compare_mapping_code.Rd @@ -103,6 +103,7 @@ Other compare: \code{\link{lnk_compare_rollup}()}, \code{\link{lnk_compare_wsg}()}, \code{\link{lnk_mapping_code}()}, -\code{\link{lnk_parity_annotate}()} +\code{\link{lnk_parity_annotate}()}, +\code{\link{lnk_rollup_wsg}()} } \concept{compare} diff --git a/man/lnk_compare_rollup.Rd b/man/lnk_compare_rollup.Rd index 6e4bec2..d6b072f 100644 --- a/man/lnk_compare_rollup.Rd +++ b/man/lnk_compare_rollup.Rd @@ -34,10 +34,13 @@ when \code{reference = "bcfishpass"} (bcfp tunnel at discovers the set from PG.} } \value{ -A tibble with one row per (species, habitat_type) — 7 -habitat types per species. Columns: \code{wsg}, \code{species}, +A tibble with one row per (species, habitat_type) — 8 +habitat types per species (the 7 habitat km/ha types plus +\code{accessible} km, link#221). Columns: \code{wsg}, \code{species}, \code{habitat_type}, \code{unit} (\code{km} | \code{ha}), \code{link_value}, \code{ref_value}, -\code{diff_pct}. +\code{diff_pct}. \code{accessible}'s \code{ref_value} is sourced tunnel-free from +\code{fresh.streams_vw_bcfp} for the salmon group (CH/CM/CO/PK/SK); other +species carry \code{NA} until their reference path lands (link#221 Phase 3). } \description{ Comparison-only counterpart to \code{\link[=lnk_pipeline_run]{lnk_pipeline_run()}}. Reads the @@ -94,6 +97,7 @@ Other compare: \code{\link{lnk_compare_mapping_code}()}, \code{\link{lnk_compare_wsg}()}, \code{\link{lnk_mapping_code}()}, -\code{\link{lnk_parity_annotate}()} +\code{\link{lnk_parity_annotate}()}, +\code{\link{lnk_rollup_wsg}()} } \concept{compare} diff --git a/man/lnk_compare_wsg.Rd b/man/lnk_compare_wsg.Rd index 1a05626..df8f34d 100644 --- a/man/lnk_compare_wsg.Rd +++ b/man/lnk_compare_wsg.Rd @@ -65,12 +65,13 @@ deprecation warning when supplied.} \value{ A list with two elements: \itemize{ -\item \code{rollup}: tibble with one row per (species, habitat_type) — 7 +\item \code{rollup}: tibble with one row per (species, habitat_type) — 8 habitat types: \code{spawning}, \code{rearing}, \code{lake_rearing}, \code{wetland_rearing}, \code{rearing_stream}, \code{rearing_lake_centerline}, -\code{rearing_wetland_centerline}. Columns: \code{wsg}, \code{species}, -\code{habitat_type}, \code{unit} (\code{km} | \code{ha}), \code{link_value}, -\code{ref_value}, \code{diff_pct}. +\code{rearing_wetland_centerline}, \code{accessible} (km, link#221). +Columns: \code{wsg}, \code{species}, \code{habitat_type}, \code{unit} (\code{km} | \code{ha}), +\code{link_value}, \code{ref_value}, \code{diff_pct}. \code{accessible}'s \code{ref_value} +is \code{NA} until the tunnel-free reference path lands. \item \code{mapping_code}: tibble with one row per species — segment-level match stats vs \code{bcfishpass.streams_mapping_code}. Columns: \code{wsg}, \code{species}, \code{total_segs}, \code{match_pct}, \code{n_diffs}, @@ -152,6 +153,7 @@ Other compare: \code{\link{lnk_compare_mapping_code}()}, \code{\link{lnk_compare_rollup}()}, \code{\link{lnk_mapping_code}()}, -\code{\link{lnk_parity_annotate}()} +\code{\link{lnk_parity_annotate}()}, +\code{\link{lnk_rollup_wsg}()} } \concept{compare} diff --git a/man/lnk_mapping_code.Rd b/man/lnk_mapping_code.Rd index 6accf15..a8879d0 100644 --- a/man/lnk_mapping_code.Rd +++ b/man/lnk_mapping_code.Rd @@ -125,6 +125,7 @@ Other compare: \code{\link{lnk_compare_mapping_code}()}, \code{\link{lnk_compare_rollup}()}, \code{\link{lnk_compare_wsg}()}, -\code{\link{lnk_parity_annotate}()} +\code{\link{lnk_parity_annotate}()}, +\code{\link{lnk_rollup_wsg}()} } \concept{compare} diff --git a/man/lnk_parity_annotate.Rd b/man/lnk_parity_annotate.Rd index 6e088b0..d8cf862 100644 --- a/man/lnk_parity_annotate.Rd +++ b/man/lnk_parity_annotate.Rd @@ -64,6 +64,7 @@ Other compare: \code{\link{lnk_compare_mapping_code}()}, \code{\link{lnk_compare_rollup}()}, \code{\link{lnk_compare_wsg}()}, -\code{\link{lnk_mapping_code}()} +\code{\link{lnk_mapping_code}()}, +\code{\link{lnk_rollup_wsg}()} } \concept{compare} diff --git a/man/lnk_pipeline_break.Rd b/man/lnk_pipeline_break.Rd index b999cf1..6c4d465 100644 --- a/man/lnk_pipeline_break.Rd +++ b/man/lnk_pipeline_break.Rd @@ -52,7 +52,7 @@ Valid entries for \code{cfg$pipeline$break_order}. Omit an entry to skip both segmentation and access-gating from that source.\tabular{llll}{ name \tab source table \tab role \tab classify label \cr \code{observations} \tab \verb{.observations_breaks} \tab fish observations from \code{bcfishobs.observations}, WSG- and species-filtered, exclusions applied \tab (informational; not a barrier) \cr - \code{gradient_minimal} \tab \verb{.gradient_barriers_minimal} \tab minimal-reduced gradient barriers (per-model 15/20/25/30\%) \tab classify uses the FULL set with \verb{gradient_} labels \cr + \code{gradient_minimal} \tab \verb{.gradient_barriers_minimal} \tab FULL per-model gradient barriers (15/20/25/30\%) + falls — every position breaks the network so access gates at each frontier (#223); NOT minimal-reduced despite the legacy name \tab classify uses the same FULL set with \verb{gradient_} labels \cr \code{falls} \tab \verb{.falls} \tab natural waterfalls from \code{whse_basemapping.fwa_obstacles_sp} (loaded by \code{prep_load_aux}); each fall is its own barrier (NOT minimal-reduced) \tab \code{blocked} \cr \code{barriers_definite} \tab \verb{.barriers_definite} \tab \code{user_barriers_definite.csv} for the AOI \tab \code{blocked} \cr \code{subsurfaceflow} \tab \verb{.barriers_subsurfaceflow} \tab FWA \verb{edge_type IN (1410, 1425)} start points; honours \code{user_barriers_definite_control}. Opt-in (only built when listed) \tab \code{blocked} \cr diff --git a/man/lnk_pipeline_prepare.Rd b/man/lnk_pipeline_prepare.Rd index 36af01e..5329ee3 100644 --- a/man/lnk_pipeline_prepare.Rd +++ b/man/lnk_pipeline_prepare.Rd @@ -75,9 +75,10 @@ enriched with \code{wscode_ltree} and \code{localcode_ltree} for \code{lnk_barrier_overrides()} to compute the per-species skip list. User-definite barriers are intentionally excluded here and consumed by later phases directly — bcfishpass parity. -\item Per-model barrier tables reduced to the minimal downstream-most -set via \code{\link[fresh:frs_barriers_minimal]{fresh::frs_barriers_minimal()}}, then unioned into -\code{gradient_barriers_minimal} for segmentation +\item Per-model barrier tables (gradient class-filtered + falls) unioned +into \code{gradient_barriers_minimal} — the FULL per-model position set +(NOT minimal-reduced, despite the legacy name), so streams break at +every frontier and per-segment access gating is correct (#223) \item Base stream segments (\code{fresh.streams}) loaded from FWA with channel width, stream order parent, GENERATED gradient / measures / length columns, and a unique \code{id_segment} @@ -93,9 +94,8 @@ Writes to (under the caller's working schema unless noted): \item \verb{.barriers_subsurfaceflow} (only when subsurfaceflow opted in via \code{cfg$pipeline$break_order}) \item \verb{.barrier_overrides} -\item \verb{.barriers_} + \verb{.barriers__min} -per-model pre/post minimal reduction -\item \verb{.gradient_barriers_minimal} (union of minimal positions) +\item \verb{.barriers_} — per-model gradient + falls positions +\item \verb{.gradient_barriers_minimal} (union of per-model raw positions) \item \code{fresh.streams} (base segments — not namespaced by AOI; fresh owns its output schema) \item \verb{.dams} (only when \code{conn_tunnel} is supplied) — pulled diff --git a/man/lnk_rollup_wsg.Rd b/man/lnk_rollup_wsg.Rd new file mode 100644 index 0000000..a835802 --- /dev/null +++ b/man/lnk_rollup_wsg.Rd @@ -0,0 +1,111 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/lnk_rollup_wsg.R +\name{lnk_rollup_wsg} +\alias{lnk_rollup_wsg} +\title{Roll up per-(WSG, species) length metrics from persisted state} +\usage{ +lnk_rollup_wsg( + conn, + aoi, + species, + schema = "fresh", + metrics = c(accessible_km = + "round(sum(length_metre) FILTER (WHERE access IN (1, 2))::numeric / 1000, 2)", + spawning_km = "round(sum(length_metre) FILTER (WHERE spawning)::numeric / 1000, 2)", + rearing_km = "round(sum(length_metre) FILTER (WHERE rearing)::numeric / 1000, 2)"), + where = NULL +) +} +\arguments{ +\item{conn}{A \link[DBI:DBIConnection-class]{DBI::DBIConnection} object (from \code{\link[=lnk_db_conn]{lnk_db_conn()}}).} + +\item{aoi}{Watershed group code (e.g. \code{"MORR"}). Uppercase 3-5 letters.} + +\item{species}{Character vector of species codes (e.g. \code{c("CO","BT")}). +Each must name existing \verb{.streams_habitat_} and +\verb{.streams_access.access_}. Restricted to alpha +characters — interpolated into identifiers, so validated to make +SQL injection structurally impossible.} + +\item{schema}{Persist schema holding \code{streams}, \code{streams_access}, +\verb{streams_habitat_}. Default \code{"fresh"}. Validated against the SQL +identifier whitelist.} + +\item{metrics}{Named character vector: names are output columns, +values are SQL aggregate expressions over the generic aliases +\code{length_metre}, \code{access}, \code{spawning}, \code{rearing}. Default emits +\code{accessible_km}, \code{spawning_km}, \code{rearing_km}. Raw SQL — trusted +caller input, like \code{frs_aggregate()}.} + +\item{where}{Character or \code{NULL}. Optional SQL predicate applied to the +per-species rows before aggregation (aliases available). Default +\code{NULL}.} +} +\value{ +A data.frame with one row per \verb{(wsg, species)} and one column +per metric. Columns: \code{wsg}, \code{species}, then \code{names(metrics)}. +} +\description{ +Reusable, predicate-driven roll-up over link's persisted per-species +tables. For each species it joins \verb{.streams} (length + edge +type) to \verb{.streams_habitat_} (spawning / rearing flags) +and \strong{left-joins} \verb{.streams_access} (per-species +\verb{access_} code) on the full PK \verb{(id_segment, watershed_group_code)} (#203), exposes the three species-varying +inputs under \strong{generic aliases} — \code{access} (int: -9 absent / 0 +blocked / 1 modelled / 2 observed), \code{spawning}, \code{rearing} (bool) — +then aggregates by \verb{(watershed_group_code, species_code)}. +} +\details{ +The \code{streams_access} join is a LEFT join: access is optional metadata +for a length roll-up. When a segment has no \code{streams_access} row +(access not yet built for the WSG), \code{access} is \code{NULL} and +\code{accessible_km} resolves to 0 — build it via +\code{lnk_pipeline_run(mapping_code = TRUE)} (or the unconditional access +phase). Length is never dropped by a missing access row, so the +habitat metrics (\code{spawning_km}, \code{rearing_km}) are unaffected. + +Because the per-species columns are aliased to fixed names, the +\code{metrics} SQL is written \strong{once}, species-agnostic — mirroring +\code{fresh::frs_aggregate()}'s \code{metrics} / \code{where} shape. Adding a species +is a \code{species} vector edit, not a query edit. + +This is a \strong{flat per-WSG \verb{GROUP BY}} — it sums whole-WSG length by +\verb{(watershed_group_code, species_code)}. It is distinct from +\code{\link[=lnk_aggregate]{lnk_aggregate()}} / \code{\link[fresh:frs_aggregate]{fresh::frs_aggregate()}}, which roll habitat up the +network \emph{upstream of individual crossings} (point-based traversal). Use +this for WSG totals; use those for per-crossing upstream summaries. + +\code{accessible_km} sums \verb{access IN (1, 2)} — link's per-species access +model on \code{streams_access}, the number validated against the +tunnel-free bcfp reference in \code{data-raw/parity_crosssection.R} +(accessible + spawning + rearing, 8 species x 11 WSGs). It +deliberately does \strong{not} use the +\code{accessible} boolean on \verb{streams_habitat_}, which carries +different (pre-gating) semantics and diverges from the access model +(MORR coho: 3424 km vs the validated 3330 km). +} +\examples{ +\dontrun{ +conn <- lnk_db_conn() +# Coho accessible / spawning / rearing km for Morice, from persisted state. +lnk_rollup_wsg(conn, aoi = "MORR", species = "CO") + +# Custom metric: count accessible segments per species. +lnk_rollup_wsg(conn, aoi = "MORR", species = c("CO", "BT"), + metrics = c(n_accessible = "COUNT(*) FILTER (WHERE access IN (1, 2))")) +} + +} +\seealso{ +\code{\link[=lnk_compare_rollup]{lnk_compare_rollup()}}, \code{\link[=lnk_aggregate]{lnk_aggregate()}}, +\code{\link[fresh:frs_aggregate]{fresh::frs_aggregate()}} + +Other compare: +\code{\link{lnk_access}()}, +\code{\link{lnk_compare_mapping_code}()}, +\code{\link{lnk_compare_rollup}()}, +\code{\link{lnk_compare_wsg}()}, +\code{\link{lnk_mapping_code}()}, +\code{\link{lnk_parity_annotate}()} +} +\concept{compare} diff --git a/planning/archive/2026-07-issue-221-223-accessible-km-parity/.gitkeep b/planning/archive/2026-07-issue-221-223-accessible-km-parity/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/planning/archive/2026-07-issue-221-223-accessible-km-parity/README.md b/planning/archive/2026-07-issue-221-223-accessible-km-parity/README.md new file mode 100644 index 0000000..48f7448 --- /dev/null +++ b/planning/archive/2026-07-issue-221-223-accessible-km-parity/README.md @@ -0,0 +1,15 @@ +## Outcome + +Combined **#221** (per-WSG `accessible_km` roll-up) + **#223** (access-segmentation-frontier fix), shipped as one PR. + +**#223 root cause:** `gradient_barriers_minimal` was fed the `fresh::frs_barriers_minimal()` downstream-most reduction as a *segmentation* source, so a stream segment straddled the accessibility frontier and its whole reach — including the blocked part above the barrier — was credited accessible. BT/ST `accessible_km` over-credited up to +40% (PCEA), +23.6% (FINA). **Fix** (one file, `lnk_pipeline_prepare.R`): union the RAW per-model gradient + falls positions into `gradient_barriers_minimal` so streams break at every frontier — mirroring the orphan path and matching bcfishpass. `frs_barriers_minimal` is now unused in link. + +**#221** added the `accessible_km` column to `lnk_compare_rollup` + a reusable `lnk_rollup_wsg()`. + +**Proof** (`research/parity_accessible_habitat_2026_07_03.md`, `data-raw/parity_crosssection.R`): across **11 WSGs × 8 species** (Peace/Fraser/Skeena/Columbia), `accessible_km` converges **44/44 within 0.05%**; spawning/rearing hold, the only 2 over-tolerance being the documented parked **BULK SK** (fresh#190 dual-rearing-lake topology). Zero regressions. Consolidated 3 accessible-km scripts into one. + +**Key learnings:** bcfp `spawning_`/`rearing_`/`access_` are coded **0/1/2/3** — use `IN (1,2)`, a bare `= 1` under-counts (cost me a false "+42% spawning divergence" scare). The fix is **habitat-neutral** (byte-identical spawn/rear pre/post). Segment count grows **2–3.5×** (inherent to bcfp-matching; intersects perf #205). + +**Follow-ups filed:** #224 (bcfp `dam_dnstr_ind` reservoir-inflow propagation), #225 (rename `gradient_barriers_minimal` → `gradient_barriers_break`), #226 (extend PARS vignette for `accessible_km`), #227 (`wsg_outlet` builder + single-WSG downstream-state guard). + +Closed by: branch `223-access-segmentation-frontier` — PR closes #221 + #223. diff --git a/planning/archive/2026-07-issue-221-223-accessible-km-parity/findings.md b/planning/archive/2026-07-issue-221-223-accessible-km-parity/findings.md new file mode 100644 index 0000000..f009d2a --- /dev/null +++ b/planning/archive/2026-07-issue-221-223-accessible-km-parity/findings.md @@ -0,0 +1,71 @@ +# Findings — accessible_km segmentation-frontier fix (#223) + +## Root cause (from research/accessible_km_divergence.md + code trace 2026-07-03) + +The BT/ST `accessible_km` over-credit is a **segmentation** defect, not an +access-decision defect. At co-located segment positions link and bcfp agree on BT +accessibility at 99.99% (`link_only` = link-acc & bcfp-blocked = 0 km). The gap is +that link's streams don't break at the per-species gradient/falls barriers, so one +segment straddles the frontier and the whole reach is credited accessible. + +## The code path + +1. `R/lnk_pipeline_run.R:216-224` — access set `barriers__access` passed to + `lnk_pipeline_access()`. Correct — holds the frontier barrier. +2. `R/lnk_pipeline_prepare.R:566-594` builds `barriers_` = class-filtered + `gradient_barriers_raw` ∪ falls, then `:592` runs `frs_barriers_minimal()`, and + `:637-651` unions results into `gradient_barriers_minimal` (the break source). +3. `fresh/R/frs_barriers_minimal.R:114-134` — DELETEs every barrier with another + downstream on the same flow path (keeps downstream-most). Correct for a decision, + wrong as a segmentation source. +4. `R/lnk_pipeline_access.R:155-165` / `:356` / `:364` — a segment is blocked only if a + barrier sits downstream of its downstream route measure. A straddling segment has + its measure below the barrier → labelled accessible. + +## Why the fix is safe + isolated + +- `frs_barriers_minimal` used **once** in link (`prepare.R:592`). +- `gradient_barriers_minimal` consumed **only** by `lnk_pipeline_break.R:110`. +- `barriers__access` built independently by `lnk_barriers_unify` + + `lnk_barriers_views.R:171` (anti-join over unified post-override barriers) — does not + touch `gradient_barriers_minimal` / `_min`. + +## Decisive evidence — blk 359209845 (FINA, BT) + +| set | count on blk | frontier 3834.78? | +|---|---|---| +| `working_fina.barriers_bt` (pre-minimal) | 16 | yes | +| `gradient_barriers_minimal` (via `_min`) | 0 | no | +| `barriers_bt_access` (decision) | 16 | yes | + +`frs_barriers_minimal` pruned all 16 tributary barriers because two BT barriers on the +parent mainstem blk 359572348 (measures 1684183.02 / 1706109.93) are downstream of the +confluence per `fwa_upstream()` — but those parents are overridden OUT of +`barriers_bt_access`, so the segmentation frontier and access frontier disagree. + +Result: link `[3391,7998]` = 4607 m accessible (whole); bcfp `[3391,3835]` 444 m +accessible + `[3835,7998]` **4163 m blocked**. Over-credit = 4163 m on this one blk. +Aggregate: FINA +1438 km, PCEA +2021 km, PARS +238 km. + +## Live pre-fix fingerprint (local fwapg :5432, 2026-07-03) + +- `working_fina.gradient_barriers_minimal` on blk 359209845 = **empty** (no break at 3835). +- `fresh.streams` blk 359209845 FINA: `id_segment 4218` = `[3391, 7998]` = **4607 m one + segment** straddling 3835 (segments 4215–4217 break normally below 3391). +- BT `accessible_km` FINA: link **7520.7 km** / bcfp **6085.2 km** = **+23.59%**. +- Post-fix expectation: blk breaks at ~3835; the segment starting there gets + `access_bt = 0`; FINA link BT km drops toward 6085. + +## bcfp reference identity + +Tunnel-free `fresh.streams_vw_bcfp`: `smnorris/bcfishpass@v0.7.15-41-g2917790` +(head_sha 29177906, date_completed 2026-07-01), `db/model/model_access_bt.sql`. + +## Encodings / gotchas + +- `access_` int: −9 absent / 0 blocked / 1 modelled / 2 observed. accessible = IN(1,2). +- bcfp predicate: `barriers__dnstr = ''` (empty string, char varying — NOT text[]). +- gradient_class: link ×10000 (0.25→2500), bcfp ×100 (0.25→25). +- #203: join persisted `fresh.*` on full PK `(id_segment, watershed_group_code)`. +- Cross-DB: :5432 (link/fresh) and :63333 (bcfp tunnel, intermittent) — no direct join. + accessible_km stays tunnel-free via `fresh.streams_vw_bcfp` on :5432. diff --git a/planning/archive/2026-07-issue-221-223-accessible-km-parity/progress.md b/planning/archive/2026-07-issue-221-223-accessible-km-parity/progress.md new file mode 100644 index 0000000..4413f44 --- /dev/null +++ b/planning/archive/2026-07-issue-221-223-accessible-km-parity/progress.md @@ -0,0 +1,85 @@ +# Progress — accessible_km segmentation-frontier fix (#223) + +## Session 2026-07-03 + +- Root-caused the BT/ST `accessible_km` over-credit to `gradient_barriers_minimal` + being fed the `frs_barriers_minimal()` downstream-most reduction as a segmentation + source (`lnk_pipeline_prepare.R:592`). Verified `frs_barriers_minimal` is single-use, + `gradient_barriers_minimal` is segmentation-only, and `barriers__access` is built + independently — so the fix is isolated. +- Filed **#223** with root-cause framing + embedded PNG (`research/blk359209845_bt_accessible_km.png`, + committed `1a26f7d` on the 221 branch; raw URL pinned for the issue embed). +- Committed the research doc + 221 findings to the 221 branch (`4549713`), pushed 221. +- Created branch `223-access-segmentation-frontier` off `origin/main`; unset its + main-tracking upstream (safety). +- User decisions: fix branch off main (own PR, closes #223); keep table name now + + file a separate rename issue once confirmed. +- Scaffolded PWF baseline (`43866a8`). +- **Confirmed the pre-fix fingerprint live** (see findings.md "Live pre-fix fingerprint"): + `working_fina.gradient_barriers_minimal` empty on blk 359209845; `fresh.streams` + id_segment 4218 = `[3391,7998]` 4607 m one segment; BT accessible_km FINA link 7520.7 + / bcfp 6085.2 = +23.59%. FINA/PARS/PCEA all persisted + in `fresh.streams_vw_bcfp`. +- Enriched issue #223 with the full code-trace, the exact fix, and the live fingerprint. + +## Handoff → new session + +Session is being handed off (context-heavy). Pick up here: +1. Read issue #223 (now comprehensive) + this PWF + `research/accessible_km_divergence.md` + (on the 221 branch). +2. Phase 1: write `data-raw/accessible_km_fix_validate.R` (segment 3835 blocked + BT + accessible_km FINA/PARS/PCEA vs bcfp + salmon clean). It should FAIL pre-fix. +3. Phase 2: the one-file fix — `lnk_pipeline_prepare.R:592-593`, union raw `model_tbl` + into `gradient_barriers_minimal` (drop `frs_barriers_minimal` for segmentation); + keep the name, update stale doc comments. +4. Phase 3: re-run FINA/PARS/PCEA (`lnk_pipeline_run(..., mapping_code = TRUE)`), + validate no-regression; `devtools::test()` + `lintr` clean. +5. Phase 4: `/code-check`, atomic commit, file the rename follow-up issue, `/gh-pr-push`. + +Tasks #16–#19 track the remaining phases. + +## Session 2026-07-03 (cont.) — Phase 1 complete + +- Reproduced the full pre-fix fingerprint live on `:5432` fwapg (`working_fina.barriers_bt` + = 16 barriers incl. frontier 3834.78; `barriers_bt_min` = 0 rows on the blk = the + `frs_barriers_minimal` smoking gun; `gradient_barriers_minimal` empty; segment 4218 + `[3390.6, 7998.1]` 4607.5 m straddling, `access_bt = 1`). +- **Discovered FINA/PARS/PCEA carry no salmon/ST** (Peace, above Bennett dam) — the plan's + item (d) "salmon (CO) ≤ tolerance" would be vacuous there. Swept all persisted WSGs and + picked **LKEL** (smallest with ST + salmon: BT 401 / ST 365 / CO 328 km; pre-fix already + clean) as the no-regression sentinel. +- Wrote `data-raw/accessible_km_fix_validate.R` (read-only, exits non-zero on any breach): + parity sweep over {FINA,PARS,PCEA,LKEL} × 8 species with a `both-present` assert gate, + plus 4 structural checks on the canonical blk 359209845. Matches `wsg_run_one.R` conn + recipe + `LNK_LOAD=loadall` guard. +- **Confirmed FAILS pre-fix** (exit 1, 7 checks): FINA/PARS/PCEA BT parity + all 4 FINA + structural; LKEL sentinel passes; bcfp-only SK correctly excluded. +- `lintr`: only house-style hanging-indent nits (siblings in `data-raw/` share them; + `lint_package()` doesn't scan `data-raw/`). No line-length breaches. +- Next: Phase 2 — the one-file fix at `lnk_pipeline_prepare.R:592-593` (union raw `model_tbl` + instead of `frs_barriers_minimal`-reduced `_min`), then re-run + re-validate. + +## Session 2026-07-03 (cont.) — Phase 2 fix + downstream-fallout assessment + +- **Fix landed:** `lnk_pipeline_prepare.R` — dropped the per-model `frs_barriers_minimal` + reduction; `gradient_barriers_minimal` now unions the RAW per-model (gradient ∪ falls) + positions (mirrors the orphan path, matches bcfp). Updated stale roxygen in prepare.R + + break.R. `frs_barriers_minimal` is now unused in link; grep confirmed no other consumer + of the `_min` tables. +- **Downstream-fallout trace (user asked "will the new barriers cause issues downstream?"):** + - Traced fresh's break machinery (subagent): `frs_break_apply` dedups (`DISTINCT`+`round`) + and drops breaks within 1 m of an existing boundary → NO zero-length segments; generated + measure columns make re-breaking idempotent; habitat length conserved under splitting. + - `classify.R:205` already builds `streams_breaks` from `gradient_barriers_raw` (FULL set) — + the fix just aligns the base segmentation to what classify already gates against. + - **Volume:** break positions explode (FINA 11→49,880; PCEA 32→82,931; PARS 10,977→64,483; + LKEL 640→5,145) → segments ~2–3.5×. Real perf/storage cost, inherent to bcfp-matching (#205). +- **LKEL canary (fix) + pre-fix baseline (stash→run→measure→pop):** + - accessible_km LKEL BT +0.72% → −0.00% exact; ST/CO/CH/CM/PK exact. Segments 3,376 → 7,446. + - habitat spawning/rearing km BYTE-IDENTICAL pre/post fix → the fix is neutral on habitat. + - mapping_code improves (711.3 → 714.1 vs bcfp 714.4). + - **Habitat parity vs bcfp is CLEAN** (BT spawning +0.0%, ST −2.6%, CO −8.7%; rearing ~+1%). + - MEASUREMENT-BUG CORRECTION (user caught it): I first reported ST/CO spawning +42%/+24% — WRONG. + `streams_vw_bcfp.spawning_` is coded 0/1/2/3; I'd filtered `= 1` and dropped the 2/3 km, + undercounting the bcfp reference. Correct predicate `spawning_ > 0`. No divergence; nothing to file. +- Next: Phase 3 — run FINA/PARS/PCEA (+ restore LKEL) with the fix, then the validator (expect + all pass); confirm BT habitat holds on the diverged WSGs. diff --git a/planning/archive/2026-07-issue-221-223-accessible-km-parity/task_plan.md b/planning/archive/2026-07-issue-221-223-accessible-km-parity/task_plan.md new file mode 100644 index 0000000..85bc79e --- /dev/null +++ b/planning/archive/2026-07-issue-221-223-accessible-km-parity/task_plan.md @@ -0,0 +1,138 @@ +# Task: BT/ST accessible_km over-credit — access segmentation drops the frontier (#223) + +The shared segmentation break network `gradient_barriers_minimal` is fed the +**minimal-reduced** per-model barriers (`lnk_pipeline_prepare.R:592-593` runs +`fresh::frs_barriers_minimal()`, which keeps only the downstream-most barrier per +flow path). Correct for an access *decision*, wrong as a *segmentation* source: it +strips interior breaks, so a link segment straddles the accessibility frontier and +the whole reach (incl. the blocked part above the barrier) is credited accessible. +Over-credit: BT `accessible_km` +23.6% (FINA), +40% (PCEA). Root-caused in +`research/accessible_km_divergence.md`; filed as #223. + +The author already applies the correct "break at every position" rule to the +**orphan** sub-threshold classes (`prepare.R:606-611`); the bug is the real +access-threshold classes don't get the same treatment. + +## Verified facts (no assumptions) +- `frs_barriers_minimal` is used **exactly once** in link (`prepare.R:592`) — only + for the per-model segmentation source. +- `gradient_barriers_minimal`'s **only** consumer is `lnk_pipeline_break.R:110` + (the `gradient_minimal` break source) — segmentation-only. +- `barriers__access` (the access **decision**) is built by a separate path: + `lnk_barriers_unify()` → `lnk_barriers_views.R:171` (anti-join over unified + post-override barriers). It does NOT derive from `gradient_barriers_minimal` or + the `_min` tables — so changing the segmentation source cannot disturb the access + decision, which already holds the frontier barrier. +- Design decision (user): keep the table name `gradient_barriers_minimal` + add a + clarifying comment now; file a **separate** rename issue once the fix is confirmed. + +## Phase 1 — Reproduce (tests-first) +- [x] Add `data-raw/accessible_km_fix_validate.R`: (a) assert `gradient_barriers_minimal` + contains the FINA/BT interior frontier (blk 359209845, ~3835); (b) assert the + segment at measure 3835 has `access_bt = 0`; (c) sum BT `accessible_km` + FINA/PARS/PCEA vs `fresh.streams_vw_bcfp` (`barriers_bt_dnstr = ''`) and assert + convergence; (d) assert salmon (CO) stays ≤ tolerance. +- [x] Run against current (pre-fix) persisted state → confirm it FAILS (bug reproduces). + +**Phase 1 result (2026-07-03):** script fails pre-fix, exit 1, 7 checks. Parity: +FINA-BT +23.59%, PARS-BT +3.43%, PCEA-BT +40.36%. Structural (blk 359209845): +gradient_barriers_minimal empty, 1 segment straddles 3834.78, no break at frontier, +accessible reach runs to 7998.1 (over-credit 4163 m — matches issue exactly). +- **Validation-set refinement:** FINA/PARS/PCEA are Peace WSGs above the Bennett dam — + BT-only, **no salmon/ST** (all 0 km), so item (d) is vacuous there. Added **LKEL** + (smallest persisted WSG with ST + salmon; pre-fix already clean BT +0.7% / ST,CO 0%) + as the no-regression sentinel. This fulfills (d), not scope creep. +- **One-sided-presence gate:** both-present → assert tolerance; a species present on only + one side FAILs by default (catches a link collapse-to-zero regression) EXCEPT the + allowlisted #189 residence cases (`residence_exclude = "LKEL:sk"` — link models CO/CH/CM/PK + not SK; bcfp lumps all salmon into one barrier group). Keeps the gate scoped to #223 + segmentation parity without masking a real regression in the exit code. +- **Code-check (2 rounds, fresh-eyes agents):** fixed 2 findings — (1) `isTRUE(all(...))` + on the S3 access_bt check so a hypothetical NULL records a clean FAIL instead of crashing; + (2) the one-sided-presence gate above (was a silent `n/a`, could have exit-0'd a regression). + Lint clean (snake_case constants) bar the shared data-raw SQL hanging-indent style. +- Reference predicate `barriers__dnstr = ''` verified identical to bcfp + `access_ IN (1,2)`. Tolerance `TOL_PCT = 1.0` (tunable in Phase 3 from post-fix numbers). +- Local fwapg is `:5432` (`postgres`/`postgres`/`fwapg`); `:63333` is the bcfp `bcfishpass` + tunnel (no `fresh.*`). Script uses the `:5432` recipe per `wsg_run_one.R`. + +## Phase 2 — Fix +- [x] `prepare.R:566-594`: union raw `barriers_` (gradient ∪ falls) into + `gradient_barriers_minimal` instead of the `frs_barriers_minimal`-reduced `_min` + table. Removed the per-model minimal call + `min_tbl`. Verified `frs_barriers_minimal` + now unused in link and no other consumer of the `_min` tables (grep). +- [x] Update stale docs (keep table name): `prepare.R` roxygen + `lnk_pipeline_break.R:24` + — now describe the FULL per-model break set. `devtools::document()` re-run. + +**LKEL canary (2026-07-03) — fix validated end-to-end, downstream fallout assessed:** +- **accessible_km converges:** LKEL BT +0.72% → **−0.00%** (exact); ST/CO/CH/CM/PK stay exact. + Segments 3,376 → 7,446 (~2.2×). Ran in 1.3 min. +- **Downstream outputs safe.** Proven by a pre-fix baseline re-run (stash → run → measure → pop): + habitat spawning/rearing km are BYTE-IDENTICAL pre-fix and post-fix (ST spawning 108.0, CO 139.2 + both runs) — the fix is neutral on habitat. mapping_code *improves* (mapping_code_bt 711.3 → 714.1 + vs bcfp 714.4). +- **Habitat parity vs bcfp is clean** (corrected measurement — see below): BT spawning +0.0%, + ST −2.6%, CO −8.7%; rearing all ~+1%. + - MEASUREMENT-BUG CORRECTION: an earlier note here claimed ST/CO spawning +42%/+24%. That was a + bad reference predicate — `fresh.streams_vw_bcfp.spawning_` is coded **0/1/2/3**, and filtering + `spawning_st = 1` dropped the 2/3 km, undercounting bcfp. Correct predicate is `spawning_ > 0`. + No such divergence exists; the fix does not touch habitat. +- **Downstream fallout verdict:** no correctness regressions from the denser break set — fresh's + break machinery dedups + drops sub-1m coincident breaks (no zero-length segments), habitat + length is conserved, and classify already gated on the FULL gradient set. The only real cost is + the 2–3.5× segment/storage/runtime increase (inherent to bcfp-matching; intersects perf issue #205). + +## Phase 3 — Validate (no-regression) +- [x] Re-ran `lnk_pipeline_run(mapping_code = TRUE)` for FINA / PARS / PCEA / LKEL. Fast: + 1.9 / 2.6 / 3.1 / 1.3 min (segments 26k→70k, 48k→97k, 34k→106k, 3.4k→7.4k). No perf blowup. +- [x] **Validation script ALL CHECKS PASS.** BT accessible_km: FINA +23.59%→**−0.02%**, + PARS +3.43%→**−0.01%**, PCEA +40.36%→**−0.01%**, LKEL exact. Structural: FINA blk 359209845 + breaks at 3835, segment above is BT-blocked, accessible reach tops at the frontier. + (Fixed validator S2: breaks round to integer via `measure_precision=0L` so the break lands + at round(3834.78)=3835 like bcfp — S2 now uses the eps window; still catches the pre-fix straddle.) +- [x] Salmon/ST no-regression (LKEL): accessible_km exact for BT/ST/CO/CH/CM/PK. +- [x] **Habitat + mapping_code parity holds** (corrected `>0` predicate): BT spawning/rearing within + ~1% on all 4 WSGs; LKEL ST spawning −2.6%, CO spawning −8.7% (modest, link-under, PRE-EXISTING + + fix-neutral); mapping_code improves. No regression from the denser segmentation. +- [x] `devtools::test()` green (`FAIL 1 | PASS 1254`; the 1 FAIL is the pre-existing + environmental `public.wsg_outlet` missing-table in test-lnk_wsg_resolve, unrelated). + Updated the 4 `.lnk_pipeline_prep_minimal` unit tests to the new contract (no + `frs_barriers_minimal`; raw `barriers_` in the union). `devtools::document()` re-run. + No new non-indentation lints on touched files. + +**Note — LKEL CO spawning −8.7%** is the one non-trivial habitat gap; it is pre-existing (identical +pre/post fix), link-conservative, and unrelated to #223 (likely the connected-waterbody spawning +nuance documented in `research/bcfishpass_methodology.md`). Out of scope; flag if it recurs broadly. + +## Phase 4 — Ship + follow-ups +- [x] `/code-check`; atomic commit (`c86f103`). +- [ ] File the separate rename issue (`gradient_barriers_minimal` → `gradient_barriers_break`). +- [x] Filed #224 — bcfp `dam_dnstr_ind` reservoir-inflow propagation (surfaced in PARS validation, out of scope). + +## Phase 5 — COMBINED #221 + #223: provincial accessible+habitat parity (one PR) +Decision (user): merge #221 (`lnk_compare_rollup` accessible_km column + `lnk_rollup_wsg`) +into #223 → **one PR closing both**. #221 surfaced #223; #223 makes #221's accessible +parity pass — inseparable for the proof. Merged 221→223 (PWF conflicts only; R/tests/research clean). +- [x] Merge `origin/221-per-wsg-habitat-access-km-rollup` into the #223 branch (`b53ad46`). +- [x] Combined package: `devtools::test()` **FAIL 1 | PASS 1298** (+44 from 221 rollup tests; the + 1 FAIL is the pre-existing environmental `public.wsg_outlet`). document() clean. +- [x] Re-ran 11-WSG cross-section with the fix (FINA/PARS/PCEA/LKEL + BULK/MORR/KISP/LFRA/USKE + + ELKR/KOTR). ~40 min local, no cyphers (BULK 5.9 / LFRA 9.4 min the largest). +- [x] Built + ran `data-raw/parity_crosssection.R` (tunnel-free; link `lnk_rollup_wsg` vs bcfp + `streams_vw_bcfp` `IN (1,2)`). **Result: accessible 44/44 exact (max 0.05%); spawn 42/43; + rear 34/35. The only 2 over-tol are BULK SK (spawn +11.0%, rear +35.2%) = documented parked + fresh#190 (Elwin+Day Lake topology). Zero #223 regressions.** #189 residence species excluded. +- [x] Wrote `research/parity_accessible_habitat_2026_07_03.md` (the accessible-column parity proof). +- [ ] `/planning-archive`; `/gh-pr-push` (PR closes #221 + #223). NEWS/DESCRIPTION bump as final commit. +- [ ] Return to the accessible_km vignette to demonstrate bcfp equivalence. + +## Validation +- `LNK_LOAD=loadall Rscript data-raw/parity_crosssection.R ` — the single + accessible+spawn+rear parity validator/proof (supersedes accessible_km_fix_validate.R + + accessible_km_proof_co.R, both folded in). Exits non-zero on any accessible-over-tol, + structural FINA-frontier failure, or non-parked habitat divergence. +- `Rscript -e 'devtools::test()' 2>&1 | grep -E "(FAIL|ERROR|PASS)"` green. + +## Out of scope +- The rename itself (separate issue, per user). +- Wiring BT/ST into `lnk_parity_annotate()` — that's #221 Phase 3, downstream of this fix. diff --git a/research/accessible_km_divergence.md b/research/accessible_km_divergence.md new file mode 100644 index 0000000..585aecc --- /dev/null +++ b/research/accessible_km_divergence.md @@ -0,0 +1,450 @@ +# accessible_km divergence — high-threshold species (BT / ST / CT-DV-RB) + +**Status:** RESOLVED + ROOT-CAUSED (2026-07-03) — mechanism is **segmentation +granularity**: link's streams don't break at gradient barriers because the break source +`gradient_barriers_minimal` is the `frs_barriers_minimal()` downstream-most reduction +(correct for the access DECISION, wrong as a SEGMENTATION source). **Fixable link +defect** — break at the full per-species barrier positions + reclassify like bcfp (do +NOT clip); `barriers__access` already holds the barrier, so no barrier-set change is +needed. See "## ROOT CAUSE" + "## RESOLVED" below. Living doc — append dated entries. +**Relates to:** link#221 (per-WSG `accessible_km` parity), `planning/active/findings.md` +(task-scoped, will be archived), and **link#223** (the tracking issue — filed 2026-07-03, +root-cause framing + embedded PNG). **Visual proof:** `research/blk359209845_bt_accessible_km.png` +(committed `1a26f7d`; generator `/tmp/blk_proof.R`) — blk 359209845 FINA BT, the +`[3835, 7998]` reach link keeps whole & accessible while bcfp blocks it. + +This doc exists because this class of problem (link-vs-bcfp parity divergence) has +historically burned tens of hours in **misdiagnosis-after-misdiagnosis**. Its +primary value is: **what has been DECISIVELY ruled out (with reproducible +evidence), so no one re-chases it**, plus the exact commands to continue. + +--- + +## TL;DR (as of 2026-07-02) + +- **Symptom:** link's `accessible_km` for BT systematically **exceeds** bcfp, up to + **+40% (PCEA)**, **+23.6% (FINA)**; **every WSG link ≥ bcfp**; magnitude scales + **monotonically** with the species `access_gradient_max` (CO 0.15 clean ≤0.27% → + ST 0.20 minor → BT 0.25 material). Extra km is **diffuse** (PCEA: ~2020 km over + 1676 blue_line_keys, no dominant BLK) and **concentrated in steep reaches**. +- **DECISIVELY RULED OUT this session:** raw gradient-barrier **placement**. On FINA, + link's ≥0.25 gradient barriers are a **strict positional superset** of bcfp's — + every bcfp barrier present in link at identical `(blue_line_key, DRM±1m)`, + `bcfp-only = 0`, link +24 (0.1%). See §"Ruled out #4". +- **Therefore the +23.6% is 100% downstream of raw gradient detection.** Narrowed to + three candidates (§"Narrowed hypotheses"): assembled barrier-**set composition**, + per-segment km **measurement/segmentation**, or the access **trace/exclusion**. +- **RESOLVED 2026-07-03 (§"## RESOLVED"):** none of the three narrowed hypotheses in + the "barrier set / access trace" family — it's **H2, segmentation granularity**. + bcfp sub-segments the identical FWA network finer than link at gradient-barrier + boundaries; every bcfp-only sub-segment is the **blocked** steep reach above the + barrier. link keeps it inside a coarser accessible segment. The over-credit km + **equals** bcfp's finer-only blocked km, 3/3 WSGs. link & bcfp agree on the access + DECISION at 99.99% of shared segment positions. +- **ROOT-CAUSED 2026-07-03 (§"## ROOT CAUSE"):** the segmentation difference is one link + code path — `gradient_barriers_minimal` (a stream-break source) is built by + `fresh::frs_barriers_minimal()` (`lnk_pipeline_prepare.R:592`), the downstream-most + per-flow-path reduction. Right for the ACCESS decision, but it strips the interior + break points, so link's segments straddle gradient barriers. **Fixable**: break at the + full barrier positions + reclassify (NOT clip) — `barriers__access` already contains + the barrier. Fix in a separate branch, then prove bcfp-equivalence in the vignette. + +--- + +## RESOLVED (2026-07-03) — segmentation-granularity length-apportionment + +**One-line:** link and bcfp agree on BT accessibility at 99.99% of co-located segment +starts; the aggregate `accessible_km` gap is **entirely** bcfp sub-segmenting the same +network finer at gradient-barrier boundaries and blocking the above-barrier tail that +link keeps whole inside a coarser accessible segment. Not a bug, not barrier placement, +not access logic, not rollup arithmetic — a **measurement granularity** difference. + +### The decisive test (per-segment confusion matrix, both DBs local on :5432) + +Both `fresh.streams`⋈`fresh.streams_access` (link) and `fresh.streams_vw_bcfp` (bcfp +ref) live on `:5432`, so they join directly on the FWA position key +`(blue_line_key, round(downstream_route_measure,3))` — the **same key** +`lnk_compare_mapping_code()` uses (`R/lnk_compare_mapping_code.R:229`). `access_bt IN +(1,2)` = link-accessible; `barriers_bt_dnstr = ''` = bcfp-accessible. + +**Link-weighted (FINA):** 26089/26094 link segments find a co-located bcfp start +(99.98% coverage; 11731/11733 km). Of that, **count-match 99.98%, km-match 99.99%**; +`link_only` (link-acc & bcfp-blocked) = **0 km**. So at shared positions the access +decision is identical — link does NOT flag co-located segments more generously. + +**Ref-weighted, all 3 WSGs** (`ref LEFT JOIN link`, bcfp segments with NO co-located +link start = bcfp's finer-only pieces): + +| WSG | agg gap `link_bt−ref_bt` | bcfp extra-seg n | extra-seg km | of which **blocked** | +|---|---:|---:|---:|---:| +| FINA | 7521−6085 = **+1436** | 2589 | 1438 | **1438** (100%) | +| PARS | 7057−6823 = **+234** | 540 | 245 | 238 (97%) | +| PCEA | 7023−5003 = **+2020** | 4258 | 2021 | **2021** (100%) | + +The over-credit **= bcfp's finer-only blocked km**, to rounding, 3/3. bcfp's extra +segments are ~99–100% blocked (they're the steep reach above the gradient barrier); +link has no break there so the reach stays in a larger accessible segment. + +### Why 99.57% mapping_code parity coexists with +23.6% accessible_km + +`lnk_compare_mapping_code()` is an **INNER** merge on position +(`R/lnk_compare_mapping_code.R:229-232`) and `match_pct = n_match/n_total` is +**segment-COUNT-weighted** (`:276`). The inner join **silently drops bcfp's finer-only +segments** — exactly the 1438 km that diverge — so parity never sees them. +accessible_km sums both full segmentations independently, so it does. Both metrics are +internally correct; they measure different populations. (Implication for #175: the +count-weighted, inner-join parity metric has a blind spot to segmentation granularity +at gradient boundaries for high-threshold species. The habitat-km rollups are unaffected +in kind but share the sensitivity.) + +### Why monotonic in `access_gradient_max` + +Higher threshold → species accesses steeper terrain → more accessible segments abut a +gradient barrier → more boundary sub-segments where link's coarser break over-credits +the above-barrier tail. CO 0.15 accesses little steep terrain (few boundaries → ≤0.27%); +BT 0.25 accesses more (many boundaries → material). Confirms the earlier USKE control +(only 6.4% of link BT-accessible km lies above 0.25 — link isn't leaking broadly; it's a +boundary-resolution effect, mean bcfp extra-seg length ~555 m on FINA). + +### Reproduce (single query, :5432, no tunnel) + +```bash +PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d fwapg -P pager=off -c " +WITH link AS ( + SELECT s.watershed_group_code AS wsg, s.blue_line_key AS blk, + round(s.downstream_route_measure::numeric,3) AS drm, + s.length_metre AS len, (a.access_bt IN (1,2)) AS link_acc + FROM fresh.streams s JOIN fresh.streams_access a + ON s.id_segment=a.id_segment AND s.watershed_group_code=a.watershed_group_code + WHERE s.watershed_group_code IN ('FINA','PARS','PCEA')), +ref AS ( + SELECT watershed_group_code AS wsg, blue_line_key AS blk, + round(downstream_route_measure::numeric,3) AS drm, + length_metre AS len, (barriers_bt_dnstr='') AS ref_acc + FROM fresh.streams_vw_bcfp WHERE watershed_group_code IN ('FINA','PARS','PCEA')) +SELECT r.wsg, + round((sum(r.len) FILTER (WHERE NOT r.ref_acc AND l.blk IS NULL)/1000.0)::numeric,0) AS unmatched_blk_km, + count(*) FILTER (WHERE l.blk IS NULL) AS unmatched_n +FROM ref r LEFT JOIN link l ON l.wsg=r.wsg AND l.blk=r.blk AND l.drm=r.drm +GROUP BY r.wsg ORDER BY r.wsg;" +``` + +Gotcha: `round(/1000.0, n)` fails (`round(double,int) does not exist`) — wrap the +whole expression `(expr)::numeric` INSIDE `round`, not the divisor. + +### Verdict + implications + +- **Not a link defect in placement or access logic.** Barriers land identically + (§"Ruled out #4"); access agrees at 99.99% of shared positions. +- **It IS a granularity accuracy gap in the *product*:** link labels ~1438 km (FINA) / + 2021 km (PCEA) of above-gradient-barrier steep reach as BT-accessible because its + segments span the barrier. Physically that reach is not BT-accessible; bcfp (finer + breaks) blocks it. For salmon (0.15) the gap is negligible (≤0.27%). +- **The reconciliation is a concrete fix, not a defer** (see §"## ROOT CAUSE"): build the + segmentation break source from the **full** per-species barrier positions instead of the + `frs_barriers_minimal()` reduction, so link's streams break at every gradient barrier + like bcfp; the existing access trace then reclassifies the above-barrier segment as + blocked. **Do NOT clip** roll-up lengths and do NOT move barriers — placement is already + identical (§"Ruled out #4") and `barriers__access` already holds the barrier. +- **#221 decision (updated):** ship salmon-group accessible_km now (proven ≤0.27%); BT/ST + unblock **after** the segmentation fix lands on its own branch — then demonstrate + bcfp-equivalence in the vignette. No longer "permanently deferred / documented + artifact"; it's a fixable defect with a known fix. +- **It's a LINK issue, not fresh.** The missing break is in link's segmentation of the + access path (`gradient_barriers_minimal`), not in fresh gradient detection. Filed as + **link#223** (2026-07-03) with the root-cause framing + embedded PNG. + +--- + +## ROOT CAUSE (code-level, 2026-07-03) — `gradient_barriers_minimal` is a minimal reduction used as a segmentation source + +The aggregate "segmentation granularity" difference above is produced by one specific +link code path. link **does** break-and-reclassify like bcfp (it does **not** clip) — but +the break source it feeds the access segmentation is the **downstream-most reduction** of +the barrier set, not the full set, so the interior break points are missing. + +### The path + +1. `lnk_pipeline_run.R:220` — the access phase runs `lnk_pipeline_access()` over the + coarse `.streams` break segmentation, deciding each segment from + `.barriers__access` (`barriers_per_sp`, assembled `:216`). +2. That `streams` segmentation is built from break sources assembled in + `lnk_pipeline_prepare.R` / `_break.R`. One source is `gradient_barriers_minimal`. +3. `lnk_pipeline_prepare.R:574-592` (`.lnk_pipeline_prep_minimal`) builds, per model, + `barriers_` = `gradient_barriers_raw WHERE gradient_class IN (class_filter)` + `UNION ALL` falls, then runs **`fresh::frs_barriers_minimal(conn, from, to)`** on it + (`:592`) and unions the per-model results into `gradient_barriers_minimal` (`:646`). +4. `frs_barriers_minimal.R:120-134` — the reduction `DELETE`s any barrier that has another + barrier **downstream** of it on the same flow path (`whse_basemapping.fwa_upstream()`), + keeping only the **downstream-most** point per path. "Once the downstream-most barrier + is present, any barrier upstream of it is redundant **for access-blocking**" — true for + the access DECISION, false for SEGMENTATION. + +Result: a segment can span a gradient barrier. Its `downstream_route_measure` sits +**below** the barrier, so `frs_network_features(direction="downstream")` in +`lnk_pipeline_access.R:156-165` finds no barrier downstream of the segment start → +`has_barriers__dnstr = FALSE` (`:356`) → `access_ = 1` (`:364`) for the **whole** +segment, including the blocked reach above the barrier. + +### The author already knew this tension — for orphans + +`lnk_pipeline_prepare.R:606-611` (comment, abridged): *"Critically — DO NOT run +frs_barriers_minimal on the orphan set. Minimal reduction keeps only the downstream-most +blocking position per flow path; that's correct semantics for ACCESS barriers, but +orphans are segmentation positions only. We want every detected position to split the +network, not just the downstream-most one."* The same reasoning applies to the per-species +gradient+falls sets that feed `gradient_barriers_minimal` — they are ALSO segmentation +positions — but the minimal reduction was left in for them. **That is the bug.** + +### Decisive evidence on blk 359209845 (FINA, BT) — the PNG segment + +Three distinct barrier sets on this blue line: + +| set | what | n on blk 359209845 | +|---|---|---:| +| `working_fina.barriers_bt` | prep, pre-minimal = gradient ≥2500 ∪ falls | **16** (incl frontier 3834.78) | +| `barriers_bt_min` (→ `gradient_barriers_minimal`) | `frs_barriers_minimal` reduction | **0** | +| `barriers_bt_access` (view) | the ACCESS-decision set | **16** (incl frontier 3834.78) | + +- **Why the minimal set is empty here:** `frs_barriers_minimal` pruned all 16 — even the + frontier at 3834.78 — because **two BT barriers on the parent mainstem blk 359572348** + (wscode `200.948755`) at measures **1684183.02 / 1706109.93** are topologically + downstream of the confluence per `fwa_upstream()`. So there is no break anywhere on + 359209845, and its top segment `[3391, 7998]` (4607 m) stays one accessible unit. +- **Why the fix needs no barrier-set change:** the ACCESS set `barriers_bt_access` + **already contains** the frontier 3834.78. Break the stream at 3835 → new segment + `[3835, 7998]` whose reference measure IS the barrier → `has_barriers_bt_dnstr = TRUE` → + reclassified **blocked**. Nothing added to the barrier set; only the segmentation changes. + +Numbers this segment asserts (matches `/tmp/blk_proof.png`): + +| | segment | length | label | +|---|---|---:|---| +| **LINK** | `[3391, 7998]` | 4607 m | access_bt=1 (accessible) — whole | +| **bcfp** | `[3391, 3835]` | 444 m | accessible | +| **bcfp** | `[3835, 7998]` | **4163 m** | **blocked** | + +Over-credit on this ONE blue line = **4163 m**. Aggregated over FINA that's the +1436 +segment / +1438 km gap in the confusion matrix above. + +### The fix (separate branch) + +Feed the access segmentation the **full** per-species barrier positions (gradient FULL ∪ +falls ∪ definite), not the `frs_barriers_minimal()` reduction — i.e. give the per-species +gradient+falls sets the same "don't-minimize, these are segmentation positions" treatment +`lnk_pipeline_prepare.R:606-611` already gives orphans. Then the existing access trace +reclassifies each above-barrier segment as blocked, exactly like bcfp. **No clipping in +the roll-up; no barrier moved.** Validate: rerun `accessible_km` for BT on FINA/PARS/PCEA +→ expect convergence to bcfp within salmon-group tolerance; then wire BT/ST into +`lnk_parity_annotate()` and demonstrate equivalence in the vignette. + +--- + +## Environment / reproducibility + +Two **separate** postgres instances (cannot SQL-join across them — see gotchas): + +| role | conn | creds | db | +|---|---|---|---| +| link/fresh local | `localhost:5432` | `postgres` / `postgres` | `fwapg` | +| bcfp tunnel | `localhost:63333` | `newgraph` / `postgres` | `bcfishpass` | + +- **Tunnel is intermittent.** Working cred is `newgraph`/`postgres` (NOT the dead + `airvine`/`*_SHARE` path in older notes). Verify live before relying on it. +- **FWA input is byte-identical** between the two DBs (USKE 23458 streams / + 9,436,876 m on both). FINA gradient-barrier class counts match to ≤0.2% (below), + independently reconfirming identical FWA. + +### Table map (what lives where) + +**link — raw gradient barriers** (per working schema; single-WSG, no `wsg` column): +`working_.gradient_barriers_raw` — `(blue_line_key, downstream_route_measure, +gradient_class, label, source, wscode_ltree, localcode_ltree)`. +`gradient_class` encoded **×10000** (0.25 → `2500`). `label='gradient'`, +`source='attribute'`. Also `working_.gradient_barriers_minimal` (island-reduced). +34 working schemas present locally (adms, bbar, …, pcea, …, ufra, unrs, will). + +**link — assembled per-species barriers** (persist, province-wide): +`fresh.barriers__unified` (VIEW: `barrier_source`, `barrier_subtype`, +`passability`, `blocks_species text[]`, `blue_line_key`, `watershed_key`, +`downstream_route_measure`, `watershed_group_code`, `geom`) and +`fresh.barriers__access`. `` ∈ {bt, ch, cm, co, pk, sk, st, …}. + +**link — km inputs** (persist): `fresh.streams` (`length_metre`; PK +`(id_segment, watershed_group_code)` — **#203: join on full PK**), +`fresh.streams_access` (`access_` int: −9 absent / 0 blocked / 1 modelled / 2 +observed), `fresh.streams_vw_bcfp` (the tunnel-free bcfp ref; `barriers__dnstr` +stored as **comma-joined varchar**, accessible predicate `= ''`). + +**bcfp tunnel — raw gradient:** `bcfishpass.gradient_barriers` — `(blue_line_key, +downstream_route_measure, wscode_ltree, localcode_ltree, watershed_group_code, +gradient_class)`. `gradient_class` encoded **×100** (0.25 → `25`), classes +{5,7,10,12,15,20,25,30}. + +**bcfp tunnel — type-mapped / assembled / components:** `barriers_gradient` +(`barrier_type='GRADIENT_NN'`), `barriers_bt` / `barriers_ch_cm_co_pk_sk` / +`barriers_st` / `barriers_ct_dv_rb` / `barriers_wct`, and the component sources +`barriers_falls`, `barriers_subsurfaceflow`, `barriers_user_definite`. +`bcfishpass.streams` stores `barriers__dnstr` as **`array[]::text[]`** (note: +different type than `streams_vw_bcfp`'s varchar). + +Fuller cross-reference: `research/bcfp_table_map.md`. + +### Species → barrier group + gradient cutoff +`inst/extdata/configs/bcfishpass/parameters_fresh.csv` (`access_gradient_max`): +CH/CM/CO/PK/SK **0.15** → `barriers_ch_cm_co_pk_sk_dnstr`; ST **0.20** → +`barriers_st_dnstr`; WCT **0.20** → `barriers_wct_dnstr`; BT **0.25** → +`barriers_bt_dnstr`; CT/DV/RB **0.25** → `barriers_ct_dv_rb_dnstr`. +bcfp side: `model_access_bt.sql` blocks BT on `GRADIENT_25 ∪ GRADIENT_30` (≥0.25); +`model_access_ch_cm_co_pk_sk.sql` on `GRADIENT_15/20/25/30` (≥0.15); +`model_access_ct_dv_rb.sql` on `GRADIENT_25/30` (≥0.25). + +--- + +## Ruled OUT (with evidence — do NOT re-chase) + +**1. Gradient-barrier detection METHOD.** fresh `.frs_break_find_multiclass` +(`fresh/R/frs_break.R`) is a faithful port of bcfp `gradient_barriers_load.sql` +(`smnorris/bcfishpass@f4ae29d`): same 100 m upstream window, same per-FWA-vertex +`ST_LocateAlong` Z-delta on `whse_basemapping.fwa_stream_networks_sp`, same +`blue_line_key = watershed_key` mainstem filter, same island grouping, `min_length=0`. + +**2. FWA input.** Byte-identical between :5432 and :63333 (USKE 23458 / +9,436,876 m). + +**3. Gradient cutoff value.** BT ≥0.25 on both sides (link +`access_gradient_max=0.25`; bcfp `GRADIENT_25 ∪ GRADIENT_30`). + +**4. Raw gradient-barrier PLACEMENT — decisively ruled out 2026-07-02.** +Class-by-class raw counts, FINA (link `gradient_class` ×10000 ↔ bcfp ×100): + +| gradient | link n | bcfp n | Δ | +|---|---|---|---| +| 0.15 | 13110 | 13085 | +25 | +| 0.20 | 13263 | 13241 | +22 | +| 0.25 | 13117 | 13101 | +16 | +| 0.30 | 10390 | 10382 | +8 | + +Positional diff of the **≥0.25** set (blk, DRM→nearest m): link **23,507**, bcfp +**23,483**, **common 23,483**, **bcfp-only 0**, **link-only 24 (0.1%)**. link's raw +gradient barriers are a **strict positional superset** of bcfp's. The 24 extra would +make link *more* restrictive — **opposite** to the observed over-credit. So raw +gradient placement cannot be the cause; the divergence is entirely downstream of it. + +Reproduce (note: `comm` needs **lexically** sorted input — psql `ORDER BY` sorts +numerically, so pipe through `sort`): +```bash +PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d fwapg -At -F',' \ + -c "SELECT blue_line_key, round(downstream_route_measure::numeric) + FROM working_fina.gradient_barriers_raw WHERE gradient_class >= 2500" \ + | sort > /tmp/fina_link_ge25.sorted +PGPASSWORD=postgres psql -h localhost -p 63333 -U newgraph -d bcfishpass -At -F',' \ + -c "SELECT blue_line_key, round(downstream_route_measure::numeric) + FROM bcfishpass.gradient_barriers + WHERE watershed_group_code='FINA' AND gradient_class >= 25" \ + | sort > /tmp/fina_bcfp_ge25.sorted +comm -12 /tmp/fina_link_ge25.sorted /tmp/fina_bcfp_ge25.sorted | wc -l # common +comm -23 /tmp/fina_link_ge25.sorted /tmp/fina_bcfp_ge25.sorted | wc -l # link-only +comm -13 /tmp/fina_link_ge25.sorted /tmp/fina_bcfp_ge25.sorted | wc -l # bcfp-only +``` + +--- + +## Narrowed hypotheses (all downstream of raw gradient detection) + +Direction constraint: **link ≥ bcfp ⇒ link applies FEWER effective downstream +barriers than bcfp** (more accessible = fewer barriers blocking upstream). Since raw +gradient barriers are identical, the missing/differing barriers or measurement is +elsewhere. + +**H1 — Assembled barrier-SET composition.** bcfp BT set +(`model_access_bt.sql`) = `gradient(≥0.25) ∪ barriers_falls ∪ barriers_subsurfaceflow +∪ barriers_user_definite`, **MINUS** barriers with a fish observation upstream (20 m +tol) or confirmed habitat upstream (200 m tol). If link's BT set omits (or +down-weights) falls/subsurfaceflow that bcfp includes, and those sit **below steep +terrain**, bcfp blocks the steep upstream habitat that link credits → matches both +the direction AND the steep-reach concentration. (SETN subsurfaceflow-stale is a +known member of this class — `research/bcfp_divergence_taxonomy.yml`, +`provincial_parity_2026_05_01.md`.) **Test:** diff `fresh.barriers_bt_unified` vs +`bcfishpass.barriers_bt` on FINA, bucketed by `barrier_source`/type. + +**H2 — Per-segment km MEASUREMENT / segmentation.** Identical barrier positions but +**different stream segmentation** → a segment straddling the 0.25 boundary gets +credited accessible **as a whole unit**. Scales with threshold (steeper terrain ⇒ +more/longer boundary segments), which fits the monotonic CO= 2500` ↔ bcfp `gradient_class in + (25,30)`). +- **#203 join discipline:** `id_segment` is NOT globally unique in the consolidated + `fresh` persist; join on full PK `(id_segment, watershed_group_code)`. +- **Tunnel cred:** `newgraph`/`postgres`, intermittent availability. + +--- + +## Session log + +### 2026-07-03 +- **Root-caused** the BT accessible_km over-credit to `gradient_barriers_minimal` being + the `frs_barriers_minimal()` downstream-most reduction used as a segmentation break + source (`lnk_pipeline_prepare.R:592`; the author already excludes orphans from this at + `:606-611`). Proven at segment level on blk 359209845 (FINA, BT): the `[3391, 7998]` + 4607 m segment stays accessible whole because the minimal reduction pruned all 16 + barriers (parent-mainstem blk 359572348 barriers downstream of the confluence), while + `barriers_bt_access` still holds the frontier 3834.78 → break+reclassify fixes it with + no barrier-set change. Over-credit on this blk = 4163 m. +- Built the visual proof (generator `/tmp/blk_proof.R`): schematic linear diagram + + geographic map, LINK vs bcfp, barrier at 3835. Committed to the repo as + `research/blk359209845_bt_accessible_km.png` (`1a26f7d`) — gists don't reliably serve + binary PNGs, so a committed file + pinned `raw.githubusercontent.com` URL is the + hosting mechanism for the issue embed. +- Verdict shifted from "defer BT/ST as documented artifact" → **fixable link defect**; fix + on a separate branch (break at full per-species barrier positions + reclassify, NOT + clip), then demonstrate bcfp-equivalence in the vignette. No pipeline code committed yet. +- **Filed link#223** (2026-07-03) with the root-cause framing + embedded PNG. Pushed the + `221-…` branch (was 8 commits ahead, never pushed) to host the image. +- Next: land the segmentation fix on its own branch. + +### 2026-07-02 +- Ruled out raw gradient-barrier placement decisively (FINA positional superset; + §Ruled-out #4). Established the link/bcfp table map + class encodings. Narrowed to + H1/H2/H3. No code committed; read-only DB work. Next: H1 assembled-set diff on FINA. diff --git a/research/blk359209845_bt_accessible_km.png b/research/blk359209845_bt_accessible_km.png new file mode 100644 index 0000000..d72dce6 Binary files /dev/null and b/research/blk359209845_bt_accessible_km.png differ diff --git a/research/parity_accessible_habitat_2026_07_03.md b/research/parity_accessible_habitat_2026_07_03.md new file mode 100644 index 0000000..ff381cd --- /dev/null +++ b/research/parity_accessible_habitat_2026_07_03.md @@ -0,0 +1,68 @@ +# Provincial parity — accessible + spawning + rearing (2026-07-03) + +Cross-section parity proof for the combined **#221** (per-WSG accessible_km roll-up) ++ **#223** (access-segmentation-frontier fix). Extends the historical spawn/rear +parity harness with the **accessible_km** column and re-runs a species/region +cross-section to prove the fix converges `accessible_km` without regressing habitat. + +## Run metadata + +- **Harness:** `data-raw/parity_crosssection.R`. link side = `lnk_rollup_wsg()` (#221); + bcfp side = tunnel-free `fresh.streams_vw_bcfp`. +- **Predicate:** `IN (1,2)` on `access_` / `spawning_` / `rearing_`. bcfp + codes these `0/1/2/3`; `IN (1,2)` is the presence predicate — a bare `= 1` + under-counts spawning/rearing (caught + corrected this session). +- **Reference:** `smnorris/bcfishpass@v0.7.15-41-g2917790`, `fresh.streams_vw_bcfp`. +- **Scope:** 11 WSGs re-run with the fix (`lnk_pipeline_run(mapping_code = TRUE)`), + spanning Peace / Fraser / Skeena / Columbia, all 8 species: + - Peace (BT-only): FINA, PARS, PCEA + - Fraser/Skeena (BT+ST+salmon): LKEL, BULK, MORR, KISP, LFRA, USKE + - Columbia/Kootenay (BT+WCT): ELKR, KOTR +- **Tolerance:** accessible ≤ 1%, habitat (spawn/rear) ≤ 5%. + +## Headline result — the fix works, no regression + +| metric | pairs | WSGs | species | max \|diff\| | within tol | +|---|---|---|---|---|---| +| **accessible** | 44 | 11 | 8 | **0.05%** | **44/44** | +| spawning | 43 | 11 | 8 | 10.99% | 42/43 | +| rearing | 35 | 11 | 6 | 35.20% | 34/35 | + +**Overall: 120/122 within tolerance.** `accessible_km` is exact everywhere +(max 0.05% across all 8 species × 11 WSGs) — that is the #223 fix, before which +FINA/PCEA were +23.6% / +40.4%. + +## The only two exceptions — both pre-existing, documented, parked + +`BULK SK` spawning **+11.0%** (link 27.1 / bcfp 24.4) and rearing **+35.2%** +(link 64.6 / bcfp 47.8). This is **fresh#190** — the Elwin + Day Lake dual-rearing-lake +topology on `blue_line_key 360846413`, documented at +`research/bcfishpass_methodology.md:124-141` (spawning +11.0% recorded there verbatim). +Pre-existing, unrelated to #223, and exactly the "if the rollup surfaces similar SK +bumps on multi-lake topologies" case that doc predicted. **No #223 regression.** + +## Excluded — species link does not model (#189 residence) + +bcfp lumps all salmon into one barrier group so its `access_` is populated +everywhere; link only models a species where `wsg_species_presence` declares it. These +are one-sided (bcfp present, link 0) and excluded from the parity assertion (tracked in +#189), not silent: + +- LKEL SK; BULK CM; MORR CM; USKE CM; USKE PK. + +## Convergence highlight (accessible_km, BT — the #223 target) + +| WSG | pre-fix | post-fix | +|---|---|---| +| FINA | +23.59% | **−0.02%** | +| PARS | +3.43% | **−0.01%** | +| PCEA | +40.36% | **−0.01%** | +| LKEL | +0.72% | **0.00%** | + +## Reproduce + +``` +# each WSG first re-run with the fix, then: +LNK_LOAD=loadall Rscript data-raw/parity_crosssection.R \ + FINA PARS PCEA LKEL BULK MORR KISP LFRA USKE ELKR KOTR +``` diff --git a/tests/testthat/test-lnk_compare_rollup.R b/tests/testthat/test-lnk_compare_rollup.R index f28378b..ec0930b 100644 --- a/tests/testthat/test-lnk_compare_rollup.R +++ b/tests/testthat/test-lnk_compare_rollup.R @@ -118,13 +118,14 @@ test_that("lnk_compare_rollup composes resolve → link-rollup → ref-rollup rearing_stream_km = 15, rearing_lake_centerline_km = 3, rearing_wetland_centerline_km = 2, + accessible_km = 18, stringsAsFactors = FALSE), lake_ha = data.frame(species_code = "BT", lake_rearing_ha = 100, stringsAsFactors = FALSE), wetland_ha = data.frame(species_code = "BT", wetland_rearing_ha = 50, stringsAsFactors = FALSE)) } - m_ref <- function(reference, conn_ref, aoi, species) { + m_ref <- function(reference, conn_ref, aoi, species, conn) { calls <<- c(calls, "ref") data.frame(species_code = "BT", spawning_km = 11, rearing_km = 21, @@ -148,10 +149,19 @@ test_that("lnk_compare_rollup composes resolve → link-rollup → ref-rollup ) expect_equal(calls, c("resolve", "link", "ref")) - # 7 habitat types × 1 species - expect_equal(nrow(result), 7L) + # 8 habitat types × 1 species (7 habitat + accessible, link#221) + expect_equal(nrow(result), 8L) expect_named(result, c("wsg", "species", "habitat_type", "unit", "link_value", "ref_value", "diff_pct")) expect_setequal(unique(result$species), "BT") expect_setequal(unique(result$wsg), "ADMS") + + # accessible row: link value flows from km$accessible_km; ref has no + # accessible column (tunnel-free ref path not yet wired) → NA diff. + acc <- result[result$habitat_type == "accessible", ] + expect_equal(nrow(acc), 1L) + expect_equal(acc$link_value, 18) + expect_equal(acc$unit, "km") + expect_true(is.na(acc$ref_value)) + expect_true(is.na(acc$diff_pct)) }) diff --git a/tests/testthat/test-lnk_compare_wsg.R b/tests/testthat/test-lnk_compare_wsg.R index d9b7bf7..35b79c1 100644 --- a/tests/testthat/test-lnk_compare_wsg.R +++ b/tests/testthat/test-lnk_compare_wsg.R @@ -169,8 +169,8 @@ test_that("lnk_compare_wsg composes lnk_pipeline_run then lnk_compare_rollup (ro habitat_type = c("spawning", "rearing", "lake_rearing", "wetland_rearing", "rearing_stream", "rearing_lake_centerline", - "rearing_wetland_centerline"), - unit = c("km", "km", "ha", "ha", "km", "km", "km"), + "rearing_wetland_centerline", "accessible"), + unit = c("km", "km", "ha", "ha", "km", "km", "km", "km"), link_value = 1, ref_value = 1, diff_pct = 0) } @@ -200,14 +200,14 @@ test_that("lnk_compare_wsg composes lnk_pipeline_run then lnk_compare_rollup (ro expect_named(result, c("rollup", "mapping_code")) expect_null(result$mapping_code) expect_s3_class(result$rollup, "tbl_df") - expect_equal(nrow(result$rollup), 7L) + expect_equal(nrow(result$rollup), 8L) }) # --------------------------------------------------------------------------- # Rollup tibble shape + diff_pct computation # --------------------------------------------------------------------------- -test_that(".lnk_compare_wsg_assemble_rollup produces 7 rows per species + correct diff_pct", { +test_that(".lnk_compare_wsg_assemble_rollup produces 8 rows per species + correct diff_pct", { link_data <- list( km = data.frame( species_code = c("BT","CH"), @@ -216,6 +216,7 @@ test_that(".lnk_compare_wsg_assemble_rollup produces 7 rows per species + correc rearing_stream_km = c(150, 80), rearing_lake_centerline_km = c(30, 10), rearing_wetland_centerline_km = c(20, 10), + accessible_km = c(180, 90), stringsAsFactors = FALSE ), lake_ha = data.frame(species_code = c("BT","CH"), @@ -242,8 +243,8 @@ test_that(".lnk_compare_wsg_assemble_rollup produces 7 rows per species + correc rollup_link = link_data, rollup_ref = ref_data ) - # 7 habitat types × 2 species - expect_equal(nrow(out), 14L) + # 8 habitat types × 2 species (7 habitat + accessible, link#221) + expect_equal(nrow(out), 16L) expect_named(out, c("wsg", "species", "habitat_type", "unit", "link_value", "ref_value", "diff_pct")) expect_setequal(unique(out$wsg), "TEST") @@ -254,6 +255,14 @@ test_that(".lnk_compare_wsg_assemble_rollup produces 7 rows per species + correc expect_equal(bt_spawn$ref_value, 99) expect_equal(bt_spawn$diff_pct, 1.0) + # accessible: link_value from km$accessible_km; ref lacks the column + # (tunnel-free ref not yet wired) → ref_value + diff_pct NA. + bt_acc <- out[out$species == "BT" & out$habitat_type == "accessible", ] + expect_equal(bt_acc$link_value, 180) + expect_equal(bt_acc$unit, "km") + expect_true(is.na(bt_acc$ref_value)) + expect_true(is.na(bt_acc$diff_pct)) + bt_lake <- out[out$species == "BT" & out$habitat_type == "lake_rearing", ] expect_equal(bt_lake$link_value, 1000) expect_equal(bt_lake$ref_value, 1000) @@ -267,6 +276,7 @@ test_that(".lnk_compare_wsg_assemble_rollup handles NA ref values (not modelled) rearing_stream_km = 180, rearing_lake_centerline_km = 15, rearing_wetland_centerline_km = 5, + accessible_km = 190, stringsAsFactors = FALSE), lake_ha = data.frame(species_code = "RB", lake_rearing_ha = 0, stringsAsFactors = FALSE), @@ -416,6 +426,7 @@ test_that(".lnk_compare_wsg_assemble_rollup handles zero ref values (avoid div-b rearing_stream_km = 180, rearing_lake_centerline_km = 0, rearing_wetland_centerline_km = 0, + accessible_km = 190, stringsAsFactors = FALSE), lake_ha = data.frame(species_code = "BT", lake_rearing_ha = 0, stringsAsFactors = FALSE), @@ -440,3 +451,52 @@ test_that(".lnk_compare_wsg_assemble_rollup handles zero ref values (avoid div-b zero_ref <- out[out$ref_value == 0, ] expect_true(all(is.na(zero_ref$diff_pct))) }) + +# --------------------------------------------------------------------------- +# Tunnel-free accessible_km reference (link#221 Phase 2 4/4) +# --------------------------------------------------------------------------- + +test_that(".lnk_compare_wsg_accessible_ref queries salmon group, short-circuits others", { + queries <- character() + m_get <- function(conn, statement, ...) { + queries <<- c(queries, statement) + data.frame(accessible_km = 3330.25) + } + + with_mocked_bindings( + dbGetQuery = m_get, + dbQuoteLiteral = function(conn, x, ...) paste0("'", x, "'"), + .package = "DBI", + { + out <- link:::.lnk_compare_wsg_accessible_ref( + conn = mock_conn(), aoi = "MORR", + species = c("CO", "BT", "SK")) + } + ) + + # CO + SK are salmon-group → one query each; BT short-circuits to NA + expect_equal(out, c(3330.25, NA_real_, 3330.25)) + expect_length(queries, 2L) + expect_true(all(grepl("barriers_ch_cm_co_pk_sk_dnstr = ''", queries))) + expect_true(all(grepl("fresh\\.streams_vw_bcfp", queries))) + expect_true(all(grepl("watershed_group_code = 'MORR'", queries))) +}) + +test_that(".lnk_compare_wsg_accessible_ref returns all-NA when no salmon species", { + m_get <- function(conn, statement, ...) { + stop("dbGetQuery should not be called for non-salmon species") + } + + with_mocked_bindings( + dbGetQuery = m_get, + dbQuoteLiteral = function(conn, x, ...) paste0("'", x, "'"), + .package = "DBI", + { + out <- link:::.lnk_compare_wsg_accessible_ref( + conn = mock_conn(), aoi = "MORR", + species = c("BT", "ST", "WCT")) + } + ) + + expect_equal(out, rep(NA_real_, 3)) +}) diff --git a/tests/testthat/test-lnk_pipeline_prepare.R b/tests/testthat/test-lnk_pipeline_prepare.R index 2b1b131..1d263b0 100644 --- a/tests/testthat/test-lnk_pipeline_prepare.R +++ b/tests/testthat/test-lnk_pipeline_prepare.R @@ -305,13 +305,18 @@ test_that(".lnk_pipeline_prep_minimal builds per-species tables from access_grad .lnk_pipeline_prep_minimal("mock-conn", aoi = "BULK", cfg = cfg, loaded = loaded, schema = "w_bulk") - # One barrier table per species (BT/CH/SK/ST/WCT) - expect_length(minimal_calls, 5L) - from_tables <- vapply(minimal_calls, `[[`, character(1), "from") - table_names <- sub("^[^.]+\\.", "", from_tables) - expect_setequal(table_names, - c("barriers_bt", "barriers_ch", "barriers_sk", - "barriers_st", "barriers_wct")) + # Fix #223: raw per-species positions feed segmentation directly — + # frs_barriers_minimal is no longer called; each raw barriers_ table + # enters the gradient_barriers_minimal union. + expect_length(minimal_calls, 0L) + union_sql <- captured[grepl( + "CREATE TABLE w_bulk\\.gradient_barriers_minimal", captured)] + expect_length(union_sql, 1L) + for (tbl in c("barriers_bt", "barriers_ch", "barriers_sk", + "barriers_st", "barriers_wct")) { + expect_match(union_sql, sprintf("FROM w_bulk\\.%s\\b", tbl)) + } + expect_no_match(union_sql, "_min\\b") # BT @ 0.25 → classes 2500, 3000 (only those ≥ 0.25) bt_create <- captured[grepl("CREATE TABLE w_bulk\\.barriers_bt\\b", captured)] @@ -335,8 +340,12 @@ test_that(".lnk_pipeline_prep_minimal builds per-species tables from access_grad }) test_that(".lnk_pipeline_prep_minimal skips species with NA / zero access_gradient_max", { + captured <- character(0) local_mocked_bindings( - .lnk_db_execute = function(conn, sql) invisible(NULL) + .lnk_db_execute = function(conn, sql) { + captured <<- c(captured, sql) + invisible(NULL) + } ) minimal_calls <- list() local_mocked_bindings( @@ -361,11 +370,16 @@ test_that(".lnk_pipeline_prep_minimal skips species with NA / zero access_gradie .lnk_pipeline_prep_minimal("mock-conn", aoi = "BULK", cfg = cfg, loaded = loaded, schema = "w") - # BT yields a barrier table; LK + ZR skipped. Orphan branch doesn't - # call frs_barriers_minimal (raw positions used directly), so only - # BT's call appears here. - expect_length(minimal_calls, 1L) - expect_match(minimal_calls[[1]]$from, "barriers_bt$") + # BT yields a barrier table; LK + ZR skipped. Fix #223: raw positions feed + # segmentation, so frs_barriers_minimal is not called — BT's raw table enters + # the union, the skipped species do not. + expect_length(minimal_calls, 0L) + union_sql <- captured[grepl( + "CREATE TABLE w\\.gradient_barriers_minimal", captured)] + expect_length(union_sql, 1L) + expect_match(union_sql, "FROM w\\.barriers_bt\\b") + expect_no_match(union_sql, "barriers_lk") + expect_no_match(union_sql, "barriers_zr") }) test_that(".lnk_pipeline_prep_minimal adds orphan-class break source when low classes present", { @@ -401,13 +415,9 @@ test_that(".lnk_pipeline_prep_minimal adds orphan-class break source when low cl .lnk_pipeline_prep_minimal("mock-conn", aoi = "BULK", cfg = cfg, loaded = loaded, schema = "w", classes = custom) - # Expected: 2 per-species tables get frs_barriers_minimal calls (BT, CH). - # Orphan branch creates barriers_orphan but does NOT call - # frs_barriers_minimal on it (raw positions used directly for segmentation). - expect_length(minimal_calls, 2L) - from_tables <- vapply(minimal_calls, `[[`, character(1), "from") - table_names <- sub("^[^.]+\\.", "", from_tables) - expect_setequal(table_names, c("barriers_bt", "barriers_ch")) + # Fix #223: no per-species frs_barriers_minimal call — the raw barriers_bt/ch + # enter the union directly, exactly as the orphan set already does. + expect_length(minimal_calls, 0L) # Orphan CREATE TABLE statement still emitted, filtering to class 500 orphan_create <- captured[grepl("CREATE TABLE w\\.barriers_orphan\\b", captured)] @@ -419,11 +429,17 @@ test_that(".lnk_pipeline_prep_minimal adds orphan-class break source when low cl union_sql <- captured[grepl("CREATE TABLE w\\.gradient_barriers_minimal", captured)] expect_length(union_sql, 1L) expect_match(union_sql, "FROM w\\.barriers_orphan\\b") + expect_match(union_sql, "FROM w\\.barriers_bt\\b") + expect_match(union_sql, "FROM w\\.barriers_ch\\b") }) test_that(".lnk_pipeline_prep_minimal does NOT add orphan branch when no low classes", { + captured <- character(0) local_mocked_bindings( - .lnk_db_execute = function(conn, sql) invisible(NULL) + .lnk_db_execute = function(conn, sql) { + captured <<- c(captured, sql) + invisible(NULL) + } ) minimal_calls <- list() local_mocked_bindings( @@ -448,10 +464,15 @@ test_that(".lnk_pipeline_prep_minimal does NOT add orphan branch when no low cla .lnk_pipeline_prep_minimal("mock-conn", aoi = "BULK", cfg = cfg, loaded = loaded, schema = "w", classes = .lnk_classes_bcfp) - from_tables <- vapply(minimal_calls, `[[`, character(1), "from") - table_names <- sub("^[^.]+\\.", "", from_tables) - expect_false("barriers_orphan" %in% table_names) - expect_setequal(table_names, c("barriers_bt", "barriers_ch")) + # Fix #223: frs_barriers_minimal not called; the union carries the raw + # per-species tables and NO orphan table (no low classes here). + expect_length(minimal_calls, 0L) + union_sql <- captured[grepl( + "CREATE TABLE w\\.gradient_barriers_minimal", captured)] + expect_length(union_sql, 1L) + expect_match(union_sql, "FROM w\\.barriers_bt\\b") + expect_match(union_sql, "FROM w\\.barriers_ch\\b") + expect_no_match(union_sql, "barriers_orphan") }) test_that(".lnk_pipeline_prep_minimal honours custom classes vector", { diff --git a/tests/testthat/test-lnk_rollup_wsg.R b/tests/testthat/test-lnk_rollup_wsg.R new file mode 100644 index 0000000..a7ab093 --- /dev/null +++ b/tests/testthat/test-lnk_rollup_wsg.R @@ -0,0 +1,112 @@ +# Tests for lnk_rollup_wsg — argument validation + SQL construction + +mock_conn <- function() structure(list(), class = "DBIConnection") + +# --------------------------------------------------------------------------- +# Argument validation +# --------------------------------------------------------------------------- + +test_that("lnk_rollup_wsg rejects invalid aoi", { + expect_error(lnk_rollup_wsg(mock_conn(), aoi = "", species = "CO"), "aoi") + expect_error(lnk_rollup_wsg(mock_conn(), aoi = "ab", species = "CO"), "aoi") + expect_error( + lnk_rollup_wsg(mock_conn(), aoi = c("MORR", "BULK"), species = "CO"), + "aoi") +}) + +test_that("lnk_rollup_wsg rejects non-DBI conn", { + expect_error(lnk_rollup_wsg("not-a-conn", aoi = "MORR", species = "CO"), + "DBI") +}) + +test_that("lnk_rollup_wsg rejects empty or non-alpha species", { + expect_error( + lnk_rollup_wsg(mock_conn(), aoi = "MORR", species = character(0)), + "species") + expect_error( + lnk_rollup_wsg(mock_conn(), aoi = "MORR", species = ""), + "species") + # Injection-shaped species suffix must be rejected before it reaches SQL. + expect_error( + lnk_rollup_wsg(mock_conn(), aoi = "MORR", species = "co; DROP TABLE x"), + "species") + expect_error( + lnk_rollup_wsg(mock_conn(), aoi = "MORR", species = "co_1"), + "species") +}) + +test_that("lnk_rollup_wsg rejects schema outside the SQL identifier whitelist", { + expect_error( + lnk_rollup_wsg(mock_conn(), aoi = "MORR", species = "CO", + schema = "Fresh"), # mixed case + "schema") + expect_error( + lnk_rollup_wsg(mock_conn(), aoi = "MORR", species = "CO", + schema = "x; DROP SCHEMA public CASCADE"), + "schema") + expect_error( + lnk_rollup_wsg(mock_conn(), aoi = "MORR", species = "CO", + schema = "1bad"), # leading digit + "schema") +}) + +test_that("lnk_rollup_wsg rejects unnamed metrics", { + expect_error( + lnk_rollup_wsg(mock_conn(), aoi = "MORR", species = "CO", + metrics = c("COUNT(*)")), + "metrics") +}) + +# --------------------------------------------------------------------------- +# SQL construction (offline, via DBI::ANSI() for standard-SQL quoting) +# --------------------------------------------------------------------------- + +test_that(".lnk_rollup_wsg_sql builds one UNION ALL branch per species", { + metrics <- c( + accessible_km = + "round(sum(length_metre) FILTER (WHERE access IN (1, 2))::numeric / 1000, 2)", # nolint: line_length_linter + spawning_km = "round(sum(length_metre) FILTER (WHERE spawning)::numeric / 1000, 2)") # nolint: line_length_linter + + sql <- link:::.lnk_rollup_wsg_sql( + conn = DBI::ANSI(), aoi = "MORR", species = c("CO", "BT"), + schema = "fresh", metrics = metrics, where = NULL) + + # Per-species table + access column, correctly lower-cased. + expect_match(sql, "fresh\\.streams_habitat_co", fixed = FALSE) + expect_match(sql, "fresh\\.streams_habitat_bt", fixed = FALSE) + expect_match(sql, "a\\.access_co AS access") + expect_match(sql, "a\\.access_bt AS access") + # species_code literal upper-cased. + expect_match(sql, "'CO' AS species_code") + expect_match(sql, "'BT' AS species_code") + # One UNION ALL joining the two branches. + expect_equal(lengths(regmatches(sql, gregexpr("UNION ALL", sql))), 1L) + # Full-PK join (#203) on both access and habitat. + expect_match(sql, "s\\.watershed_group_code = a\\.watershed_group_code") + expect_match(sql, "s\\.watershed_group_code = h\\.watershed_group_code") + # streams_access is LEFT-joined (optional) so habitat length is never + # dropped when access is unbuilt; habitat is an inner join. + expect_match(sql, "LEFT JOIN fresh\\.streams_access") + expect_match(sql, "JOIN fresh\\.streams_habitat_co") + # Metric aliases + grouping. + expect_match(sql, "AS accessible_km") + expect_match(sql, "AS spawning_km") + expect_match(sql, "GROUP BY watershed_group_code, species_code") + # aoi literal bound. + expect_match(sql, "'MORR'") +}) + +test_that(".lnk_rollup_wsg_sql appends an optional where predicate", { + metrics <- c(n = "COUNT(*)") + + sql_no_where <- link:::.lnk_rollup_wsg_sql( + conn = DBI::ANSI(), aoi = "MORR", species = "CO", + schema = "fresh", metrics = metrics, where = NULL) + expect_false(grepl("access IN \\(1, 2\\)", sql_no_where)) + + sql_where <- link:::.lnk_rollup_wsg_sql( + conn = DBI::ANSI(), aoi = "MORR", species = "CO", + schema = "fresh", metrics = metrics, where = "access IN (1, 2)") + # Outer WHERE on the aggregated subquery, before GROUP BY. + expect_match(sql_where, "per_species\\s*\\n\\s*WHERE access IN \\(1, 2\\)") +})