From 870625b2430672071bb12b926f946fe3ead012a0 Mon Sep 17 00:00:00 2001 From: Curtis Anderson Date: Wed, 22 Jul 2026 12:12:01 -0700 Subject: [PATCH] fix(storage#3.1.2): grandfather pre-3.0.43 datasets in dataset-size check Datasets generated before the floor->ceil datagen alignment (commit f9d414d, first shipped in mlpstorage 3.0.43) were sized with integer floor division in rules/utils.py. Rule 3.1.2 in the submission checker now re-derives the threshold with ceil, so it rejects a valid pre-alignment dataset by exactly one file (a submitter hit "actual files 4710038 < minimum required 4710039" on a Jun-28 run made with 3.0.24). Relax the 3.1.2 threshold to ceil-minus-one for the v3.0 round. We can't just floor it: datagen floored via integer `//` while this check recomputes the same formula as a float chain, so float drift can push floor() one above the recorded count and still reject. ceil-1 equals floor() for non-integer ratios and grants one file of slack exactly at the integer boundary where that drift bites, absorbing the bounded <=1-file divergence. Stays int-vs-int (no "N < N" display bug) and still rejects genuinely undersized runs (short by >1 file). Validator-only change; rules/utils.py (datagen/runtime verifier) stays on ceil. Revert to plain ceil once the v3.0 round closes -- post-3.0.43 datagen ceils, so newer datasets pass either way. --- .../checks/training_checks.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/mlpstorage_py/submission_checker/checks/training_checks.py b/mlpstorage_py/submission_checker/checks/training_checks.py index 92cd7062..b24e1936 100644 --- a/mlpstorage_py/submission_checker/checks/training_checks.py +++ b/mlpstorage_py/submission_checker/checks/training_checks.py @@ -193,14 +193,24 @@ def recalculate_dataset_size(self): min_samples_memory = (total_host_memory * HOST_MEMORY_MULTIPLIER * 1024 * 1024 * 1024 / record_length) - # Take max of both constraints. The memory-derived branch is - # a float, so ceil (not floor/int cast) to stay in lockstep - # with datagen (rules/utils.py) — otherwise a dataset sized - # to the floor passes runtime but fails this check by a - # single file, and the message would read "N < N" because - # int(min_total_files) truncates the float. + # v3.0-round grandfather: ceil-minus-one, not ceil. Datasets + # generated before the floor->ceil datagen alignment (mlpstorage + # <3.0.43, commit f9d414d) were sized with integer floor division + # in rules/utils.py, so a ceil threshold here rejects a valid + # pre-alignment dataset by exactly one file (e.g. 4710038 < + # 4710039). We can't just floor this expression: datagen floored + # via integer `//` while this check recomputes the same formula + # as a float chain, so float drift can push floor() one above the + # recorded count and still reject. `ceil-1` equals floor() for + # non-integer ratios and grants one extra file of slack exactly at + # the integer boundary where that drift bites — absorbing the + # bounded <=1-file divergence. Stays int-vs-int (no "N < N" + # display bug) and still rejects genuinely undersized runs (short + # by >1 file). REVERT to plain math.ceil once the v3.0 round + # closes — post-alignment datagen ceils, so newer datasets pass + # either way. See memory grandfather-floor-3.1.2-revert. min_samples = max(min_samples_steps, min_samples_memory) - min_total_files = math.ceil(min_samples / num_samples_per_file) + min_total_files = math.ceil(min_samples / num_samples_per_file) - 1 min_files_size_gb = min_samples * record_length / 1024 / 1024 / 1024 # Verify actual matches expected