diff --git a/genleaves.sh b/genleaves.sh index 862cc42..161c9b2 100755 --- a/genleaves.sh +++ b/genleaves.sh @@ -5,8 +5,6 @@ set -euo pipefail full_mode="" klv1_mode="" logs_mode="" -smooth_mode="" -no_smooth_mode="" gilles_mode="" no_forcing_mode="" while :; do @@ -29,16 +27,6 @@ while :; do shift continue fi - if [ "${1:-}" = "--smooth" ]; then - smooth_mode=1 - shift - continue - fi - if [ "${1:-}" = "--no-smooth" ]; then - no_smooth_mode=1 - shift - continue - fi if [ "${1:-}" = "--gilles" ]; then gilles_mode=1 shift @@ -99,8 +87,6 @@ options: --full generate full-rack leaves --klv1 use klv1 instead of klv2 (not recommended) --logs log complete games (not recommended if not needed) - --smooth enable smoothing (default without :min_samples_per_rack or with :0) - --no-smooth disable smoothing (default with :min_samples_per_rack) --no-forcing disable full-rack forcing (on by default for autoplay). by default the autoplay path forces each undersampled full rack to the per-gen :min (defaulting :min to 1 = cover each @@ -112,8 +98,7 @@ options: mandatory games, remediation games direct their samples at racks still seen fewer than that many times until every rack reaches it (or no further progress is possible). :0 (or omitted) is pure - board sampling. like autoplay, a nonzero :min_samples_per_rack - defaults to no smoothing; use --smooth to override + board sampling. EOF exit 2 fi @@ -183,10 +168,7 @@ last_leave="-" # continue from previous run if found while :; do - if [ -e "leaves-smooth$[num_processed + 1].${klv_ext}" ]; then - last_leave="leaves-smooth$[num_processed + 1].${klv_ext}" - let num_processed=num_processed+1 - elif [ -e "leaves$[num_processed + 1].${klv_ext}" ]; then + if [ -e "leaves$[num_processed + 1].${klv_ext}" ]; then last_leave="leaves$[num_processed + 1].${klv_ext}" let num_processed=num_processed+1 else @@ -210,24 +192,11 @@ while [ "${!i:-}" != "" ]; do fi fi - should_smooth=0 - if [ "$after_colon" = "0" ]; then - should_smooth=1 - fi - if [ "$smooth_mode" ]; then - should_smooth=1 - fi - if [ "$no_smooth_mode" ]; then - should_smooth=0 - fi - + # every generation smooths: a leave too thinly sampled to trust borrows from its + # neighbors, and a well-sampled one keeps its own average untouched, so there is + # nothing left to turn off. leaves needs no qualifier now that there is one kind. effective_generate_subcommand="${generate_subcommand}" - leave_name="leaves-smooth" - if [ "$should_smooth" = "0" ]; then - # this must come after full_mode - effective_generate_subcommand="${generate_subcommand}-no-smooth" - leave_name="leaves" - fi + leave_name="leaves" # rare-subrack coverage summary (only produced by autoplay when :min_samples > 0). # left empty when forcing is off, so the gilles path and the no-forcing diff --git a/lab/leavegen.txt b/lab/leavegen.txt index 610b17c..5a3a2b9 100644 --- a/lab/leavegen.txt +++ b/lab/leavegen.txt @@ -93,11 +93,10 @@ The third step turns that tally into a leave table, valuing each leave from the racks that hold it. Each leave's value is the average of the best-play values of the fuller racks that contain it, where a rack counts more the more often it came up and the more ways its extra tiles could have been drawn from the bag. -The whole table is then shifted so the empty leave is worth zero. There are two -variants -- smoothed and not (the difference is discussed below): +The whole table is then shifted so the empty leave is worth zero. Leaves too +thinly sampled to trust borrow their value from their neighbors on the way out +(discussed below): - cargo run --release --bin leave -- \ - english-generate-no-smooth summary.csv leaves.csv cargo run --release --bin leave -- \ english-generate summary.csv leaves.csv @@ -182,31 +181,53 @@ against a self-play table (confidence 100), and null against null is 50.00 by symmetry. -SMOOTHING, OR NOT - -The generate step comes in two variants. The plain english-generate smooths the -table: a rack seen only a handful of times has a noisy average, so its value is -pulled toward what similar, better-sampled racks say, steadying the leaves that -self-play rarely deals. english-generate-no-smooth skips that and uses the -observed averages as they stand. - -Which of the two you want depends entirely on how many games you played, and -the honest answer is that smoothing either wins outright or does nothing at -all. It is never the thing that costs you. At fifty thousand games, where four -leaves in five are too thin to trust, the smoothed table beats the unsmoothed -one about 51.6 to 48.4 over a million game pairs -- a crushing margin we are -certain of, not a coin flip. At two hundred thousand games its lead narrows to -50.3 against 49.7, still certain. At twenty million games nothing is thin any -more, nothing gets smoothed, and the two commands write a byte-for-byte -identical table. The advantage fades exactly as the thin leaves do: 89 percent -of leaves smoothed at fifty thousand games, 76 percent at two hundred -thousand, none at twenty million. - -So the choice only bites on small runs, and there it is not really a choice. -Below a certain volume the unsmoothed table is not merely noisier -- for the -leaves nothing was ever dealt for, it has no value to report at all, and -something has to be invented regardless. Smoothing is the default genleaves.sh -uses; it turns off only when a per-generation min-samples count is asked for. +SMOOTHING + +The generate step steadies the leaves it could not measure well. A rack seen +only a handful of times has a noisy average, so its value is pulled toward what +similar, better-sampled racks say; a rack no game ever dealt has no average at +all, and its value is filled in from those neighbors outright. A rack the games +covered properly keeps exactly the average they measured, untouched. + +There used to be a second variant, english-generate-no-smooth, that skipped all +of that and used the observed averages as they stood. It is gone, because +measuring it left nothing for it to be good at. Smoothing either wins outright +or does nothing whatsoever, and it never costs anything. Sweeping the number of +self-play games, each table played its unsmoothed twin over a million game +pairs: + + games leaves smoothed smoothed side won + 50,000 89% 51.61 certain + 200,000 76% 50.32 certain + 1,000,000 52% 50.03 not resolved + 5,000,000 26% 50.00 dead level + 20,000,000 0% identical file + +Every row landed on the same side of 50 and not one below it. The lead fades +exactly as the thin leaves do, and the last two rows are the telling ones: at +five million games a quarter of the table still gets smoothed and it plays +dead level, so smoothing the thin tail is not merely harmless once it stops +firing -- it is free even while it fires. At twenty million nothing is thin, so +nothing is smoothed and the two commands write a byte-for-byte identical file. + +So there was never a run on which you wanted the unsmoothed table. On a large +one it gave you the same file; on a small one it gave you a worse one, and for +the leaves nothing was ever dealt for it had nothing to report at all -- +something had to be invented regardless, so "no smoothing" was never quite +truthful. A knob whose every setting is either identical or worse is not a +choice, it is a way to make a mistake, so the variant and genleaves.sh's +--smooth / --no-smooth flags are retired. + +What is left is which leaves count as too thin, and that is worth tuning: see +WOLGES_GENERATE_SMOOTH_MIN and WOLGES_GENERATE_SMOOTH_CI below. Setting the +floor to 0 keeps every average that was actually measured, however few samples +stand behind it, and leaves only the racks nothing was ever dealt for to borrow +a value. That is close to what the retired variant did, but it is not the same +thing: that one sent every unmeasured leave to the fallback instead, which +averages the subracks one tile shorter rather than the neighbors one tile +swapped, so the two disagree on the roughly two leaves in five a small run +never deals at all. Neither is "no invented values" -- there is no such option, +because a leave no game dealt has nothing to report. (An older reading of this section had the two as a tie, at about 49.99 and 57 percent confidence. That was measured when smoothing still leaned on the @@ -338,8 +359,7 @@ mostly from one-of-a-kind tiles -- a rack that wants the Q, say -- is hard to draw, so across a whole run it turns up only a handful of times, or never. Its leaves then rest on very few samples or none, and english-generate has to guess their value from the fuller racks it did see rather than measure it directly. -(The plain english-generate does exactly that filling-in, called smoothing; the -english-generate-no-smooth variant leaves an unmeasured leave alone instead.) +(english-generate does exactly that filling-in, called smoothing.) The question this raises: if we spent the effort to sample those rare racks enough to measure them, would the resulting table play any better? @@ -1974,8 +1994,9 @@ Decompose (english-generate): fewer sampled racks behind it than this is smoothed: its value is borrowed from its one-tile-swap neighbors. Support counts samples taken, not how often the leave can be drawn, so a well-covered leave is never smoothed - however common it is. Raise it to smooth more of the thin tail; the - no-smooth variant of english-generate ignores it. + however common it is. Raise it to smooth more of the thin tail; set it to 0 + to keep every measured average and leave only the racks nothing was dealt + for to borrow one. Shared across builders: WOLGES_APPORTION (full-rack or entering; default full-rack) -- how a board's diff --git a/src/main_leave.rs b/src/main_leave.rs index ec91642..a157f16 100644 --- a/src/main_leave.rs +++ b/src/main_leave.rs @@ -745,30 +745,8 @@ fn do_lang_kwg game_config::GameConfig, N: kwg::Node + )?; Ok(true) } - "-generate-no-smooth" => { - generate_leaves::<_, _, false, false>( - make_game_config(), - csv::ReaderBuilder::new() - .has_headers(false) - .from_reader(make_reader(&args[2])?), - csv::Writer::from_writer(make_writer(&args[3])?), - args.get(4).map(|x| x.as_str()), - )?; - Ok(true) - } "-generate" => { - generate_leaves::<_, _, true, false>( - make_game_config(), - csv::ReaderBuilder::new() - .has_headers(false) - .from_reader(make_reader(&args[2])?), - csv::Writer::from_writer(make_writer(&args[3])?), - args.get(4).map(|x| x.as_str()), - )?; - Ok(true) - } - "-generate-full-no-smooth" => { - generate_leaves::<_, _, false, true>( + generate_leaves::<_, _, false>( make_game_config(), csv::ReaderBuilder::new() .has_headers(false) @@ -779,7 +757,7 @@ fn do_lang_kwg game_config::GameConfig, N: kwg::Node + Ok(true) } "-generate-full" => { - generate_leaves::<_, _, true, true>( + generate_leaves::<_, _, true>( make_game_config(), csv::ReaderBuilder::new() .has_headers(false) @@ -851,14 +829,13 @@ fn main() -> error::Returns<()> { summarize logfile into summary.csv english-resummarize concatenated_summaries.csv summary.csv combine multiple summaries into one summary.csv and recompute totals - english-generate-no-smooth summary.csv leaves.csv [rare.csv] - generate leaves (no smoothing) up to rack_size - 1 english-generate summary.csv leaves.csv [rare.csv] - generate leaves (with smoothing) up to rack_size - 1 - english-generate-full-no-smooth summary.csv leaves.csv [rare.csv] - generate leaves (no smoothing) up to rack_size + generate leaves up to rack_size - 1 english-generate-full summary.csv leaves.csv [rare.csv] - generate leaves (with smoothing) up to rack_size + generate leaves up to rack_size + a leave too thinly sampled to trust borrows its value from its + one-tile-swap neighbors; tune which those are via + WOLGES_GENERATE_SMOOTH_MIN / _CI [rare.csv] on any -generate adds direct coverage for undersampled subracks english-playability CSW24.kwg leave.klv 1000000 [seed] autoplay (not saved) and record prorated found best words (at the end) @@ -7130,12 +7107,7 @@ fn decompose_contribution(fv: &Cumulate, w: u64, per_rack: bool) -> (f64, u64) { } } -fn generate_leaves< - Readable: std::io::Read, - W: std::io::Write, - const DO_SMOOTHING: bool, - const IS_FULL_RACK: bool, ->( +fn generate_leaves( game_config: game_config::GameConfig, mut csv_in: csv::Reader, mut csv_out: csv::Writer, @@ -7276,18 +7248,16 @@ fn generate_leaves< // spread is tracked in subrack_raw below instead. sumsq: 0.0, }); - if DO_SMOOTHING { - // raw support: this rack adds its own sampled count to every - // subrack it covers, independent of the draw-ways weight above. - *subrack_support.entry(subrack_bytes.into()).or_insert(0u64) += fv.count; - // and its raw sum and squares, so the gate can weigh how tightly - // the samples agree rather than only how many there are. - let e = subrack_raw - .entry(subrack_bytes.into()) - .or_insert((0.0f64, 0.0f64)); - e.0 += fv.equity; - e.1 += fv.sumsq; - } + // raw support: this rack adds its own sampled count to every + // subrack it covers, independent of the draw-ways weight above. + *subrack_support.entry(subrack_bytes.into()).or_insert(0u64) += fv.count; + // and its raw sum and squares, so the gate can weigh how tightly + // the samples agree rather than only how many there are. + let e = subrack_raw + .entry(subrack_bytes.into()) + .or_insert((0.0f64, 0.0f64)); + e.0 += fv.equity; + e.1 += fv.sumsq; }, rack_tally: &mut rack_tally, min_len: 0, @@ -7334,14 +7304,12 @@ fn generate_leaves< let sumsq = f64::NAN; parse_rack(&rack_reader, &record[0], &mut rack_bytes)?; pool_rare_one(&mut subrack_map, &rack_bytes, equity, count, sumsq); - if DO_SMOOTHING { - *subrack_support.entry(rack_bytes[..].into()).or_insert(0u64) += count; - let e = subrack_raw - .entry(rack_bytes[..].into()) - .or_insert((0.0f64, 0.0f64)); - e.0 += equity; - e.1 += sumsq; - } + *subrack_support.entry(rack_bytes[..].into()).or_insert(0u64) += count; + let e = subrack_raw + .entry(rack_bytes[..].into()) + .or_insert((0.0f64, 0.0f64)); + e.0 += equity; + e.1 += sumsq; } } @@ -7350,13 +7318,14 @@ fn generate_leaves< // draw-ways-weighted count in subrack_map -- is below this floor. Keying on samples // means a well-sampled leave is never smoothed no matter how common; the old // cube-root-of-the-total rule smoothed most leaves even at full coverage. Default 50; - // raise to smooth more of the thin tail, lower (or --no-smooth) to trust the sampled - // means as they are. - let smooth_min = if DO_SMOOTHING { - env_usize("WOLGES_GENERATE_SMOOTH_MIN", 50) as u64 - } else { - 0 - }; + // raise to smooth more of the thin tail, lower to trust more of the sampled means as + // they stand. Set it to 0 and every average that was actually measured is kept, + // however few samples stand behind it; only a rack nothing was ever dealt for still + // borrows, because it has nothing of its own to report. That is NOT the retired + // no-smoothing variant: that one sent every unmeasured leave to the length-shorter + // fallback below instead of the one-tile-swap borrow, so the two part company over + // the leaves a small run never deals -- about two in five of them. + let smooth_min = env_usize("WOLGES_GENERATE_SMOOTH_MIN", 50) as u64; // WOLGES_GENERATE_SMOOTH_CI (a width in points; default 0.0, off) -- the principled // form of the same question. A sample floor answers "enough samples?" with a bare // count, which is a stand-in for what actually matters: is this leave's mean pinned @@ -7366,11 +7335,7 @@ fn generate_leaves< // .99), the same precision test the census applies to its own leaves. Needs the // squares column: summaries written before it existed carry no spread (NAN), and // those leaves fall back to the sample floor rather than guess. - let smooth_ci = if DO_SMOOTHING { - env_parse::("WOLGES_GENERATE_SMOOTH_CI", 0.0) - } else { - 0.0 - }; + let smooth_ci = env_parse::("WOLGES_GENERATE_SMOOTH_CI", 0.0); let smooth_ci_conf = env_parse::("WOLGES_GENERATE_SMOOTH_CONF", 0.99); let smooth_ci_conf = if smooth_ci_conf > 0.0 && smooth_ci_conf < 1.0 { smooth_ci_conf @@ -7421,21 +7386,15 @@ fn generate_leaves< let mut alphabet_freqs = (0..game_config.alphabet().len()) .map(|tile| game_config.alphabet().freq(tile)) .collect::>(); - let mut neighbor_buffer = if DO_SMOOTHING { - Vec::with_capacity(game_config.rack_size() as usize) - } else { - Vec::new() - }; + let mut neighbor_buffer = Vec::with_capacity(game_config.rack_size() as usize); let mut num_smoothed = 0u64; generate_exchanges(&mut ExchangeEnv { found_exchange_move: |rack_bytes: &[u8]| { let mut new_v = if let Some(v) = subrack_map.get(rack_bytes) { - if !DO_SMOOTHING - || well_sampled( - rack_bytes, - subrack_support.get(rack_bytes).copied().unwrap_or(0), - ) - { + if well_sampled( + rack_bytes, + subrack_support.get(rack_bytes).copied().unwrap_or(0), + ) { v.equity / v.count as f64 } else { // perform smoothing if there are too few samples. @@ -7444,7 +7403,7 @@ fn generate_leaves< } else { f64::NAN }; - if DO_SMOOTHING && new_v.is_nan() { + if new_v.is_nan() { rack_tally.iter_mut().for_each(|m| *m = 0); rack_bytes .iter()