Skip to content

Commit 225d323

Browse files
JoeMattclaude
andcommitted
Add determinism and frameskip invariance tests
- Determinism test: runs each ROM twice and verifies identical screenshots, catching any non-deterministic behavior (e.g., rand() in hot paths like the old HC register stub). - Frameskip invariance test: runs with frameskip=0 and frameskip=3 via miniretro's --envvar flag, verifies the rendered frames are identical (emulation runs every frame regardless of skip). - Extract run_and_get_frame() helper to reduce duplication. - Fix echo -e to printf '%b' and quote paths in baseline hint. Co-Authored-By: Claude Opus 4.6 <[email protected]>
1 parent ec991b1 commit 225d323

1 file changed

Lines changed: 78 additions & 2 deletions

File tree

test/regression_test.sh

Lines changed: 78 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,12 +160,88 @@ for rom in "${ROM_DIR}"/*.j64 "${ROM_DIR}"/*.rom; do
160160
fi
161161
else
162162
echo " NEW: ${rom_name} — no baseline yet"
163-
echo " To create: cp ${frame_file} ${baseline_png}"
163+
echo " To create: cp \"${frame_file}\" \"${baseline_png}\""
164164
NEW=$((NEW + 1))
165165
SUMMARY="${SUMMARY}| ${rom_name} | :new: NEW | no baseline | - |\n"
166166
fi
167167
done
168168

169+
# --- Helper: run ROM and return last screenshot path ---
170+
run_and_get_frame() {
171+
local out_dir="$1" envvar_args="$2"
172+
shift 2
173+
# shellcheck disable=SC2086
174+
"${MINIRETRO_BIN}" \
175+
--core "${CORE}" --rom "$1" \
176+
--output "${out_dir}" --system "${out_dir}" \
177+
--frames 300 --dump-frames-every 100 \
178+
--no-alarm ${envvar_args} >/dev/null 2>&1 || true
179+
find "${out_dir}" -name "screenshot*.png" 2>/dev/null | sort | tail -1
180+
}
181+
182+
# --- Determinism test: run each ROM twice, verify identical output ---
183+
# Validates that emulation is fully deterministic (no rand() in hot paths).
184+
echo ""
185+
echo "==> Running determinism check..."
186+
for rom in "${ROM_DIR}"/*.j64 "${ROM_DIR}"/*.rom; do
187+
[ -f "${rom}" ] || continue
188+
rom_name="$(basename "${rom}" | sed 's/\.[^.]*$//')"
189+
det_dir1="${WORK_DIR}/det1/${rom_name}"
190+
det_dir2="${WORK_DIR}/det2/${rom_name}"
191+
mkdir -p "${det_dir1}" "${det_dir2}"
192+
193+
frame1=$(run_and_get_frame "${det_dir1}" "" "${rom}")
194+
frame2=$(run_and_get_frame "${det_dir2}" "" "${rom}")
195+
196+
if [ -n "${frame1}" ] && [ -n "${frame2}" ]; then
197+
if cmp -s "${frame1}" "${frame2}"; then
198+
echo " PASS: ${rom_name} determinism (identical across runs)"
199+
PASS=$((PASS + 1))
200+
SUMMARY="${SUMMARY}| ${rom_name} (determinism) | :white_check_mark: PASS | identical across runs | - |\n"
201+
else
202+
echo " FAIL: ${rom_name} determinism (output differs between runs)"
203+
cp "${frame1}" "${DIFF_DIR}/${rom_name}_det_run1.png"
204+
cp "${frame2}" "${DIFF_DIR}/${rom_name}_det_run2.png"
205+
FAIL=$((FAIL + 1))
206+
SUMMARY="${SUMMARY}| ${rom_name} (determinism) | :x: FAIL | non-deterministic output | See artifacts |\n"
207+
fi
208+
else
209+
echo " SKIP: ${rom_name} determinism (no frames produced)"
210+
fi
211+
done
212+
213+
# --- Frameskip test: verify core options don't affect emulation output ---
214+
# With frameskip, video_cb receives NULL on skipped frames but the
215+
# emulation still runs identically. The last rendered frame should match.
216+
echo ""
217+
echo "==> Running frameskip invariance check..."
218+
for rom in "${ROM_DIR}"/*.j64 "${ROM_DIR}"/*.rom; do
219+
[ -f "${rom}" ] || continue
220+
rom_name="$(basename "${rom}" | sed 's/\.[^.]*$//')"
221+
fs0_dir="${WORK_DIR}/fs0/${rom_name}"
222+
fs3_dir="${WORK_DIR}/fs3/${rom_name}"
223+
mkdir -p "${fs0_dir}" "${fs3_dir}"
224+
225+
frame_fs0=$(run_and_get_frame "${fs0_dir}" "" "${rom}")
226+
frame_fs3=$(run_and_get_frame "${fs3_dir}" "--envvar virtualjaguar_frameskip=3" "${rom}")
227+
228+
if [ -n "${frame_fs0}" ] && [ -n "${frame_fs3}" ]; then
229+
if cmp -s "${frame_fs0}" "${frame_fs3}"; then
230+
echo " PASS: ${rom_name} frameskip invariance (skip=0 matches skip=3)"
231+
PASS=$((PASS + 1))
232+
SUMMARY="${SUMMARY}| ${rom_name} (frameskip) | :white_check_mark: PASS | skip=0 matches skip=3 | - |\n"
233+
else
234+
echo " FAIL: ${rom_name} frameskip invariance (output differs with frameskip)"
235+
cp "${frame_fs0}" "${DIFF_DIR}/${rom_name}_fs0.png"
236+
cp "${frame_fs3}" "${DIFF_DIR}/${rom_name}_fs3.png"
237+
FAIL=$((FAIL + 1))
238+
SUMMARY="${SUMMARY}| ${rom_name} (frameskip) | :x: FAIL | frameskip changes output | See artifacts |\n"
239+
fi
240+
else
241+
echo " SKIP: ${rom_name} frameskip (no frames produced)"
242+
fi
243+
done
244+
169245
echo ""
170246
echo "==> Results: ${PASS} passed, ${FAIL} failed, ${NEW} new (no baseline)"
171247

@@ -175,7 +251,7 @@ cat > "${DIFF_DIR}/summary.md" <<EOSUMMARY
175251
176252
| ROM | Status | Details | Diff |
177253
|-----|--------|---------|------|
178-
$(echo -e "${SUMMARY}")
254+
$(printf '%b' "${SUMMARY}")
179255
180256
**Platform:** $(uname -s) $(uname -m)
181257
EOSUMMARY

0 commit comments

Comments
 (0)