diff --git a/app/bin/dart_arena_headless.dart b/app/bin/dart_arena_headless.dart index 0fb9d80..631de99 100644 --- a/app/bin/dart_arena_headless.dart +++ b/app/bin/dart_arena_headless.dart @@ -80,6 +80,7 @@ Map _helpJson() { 'dart run --verbosity=error dart_arena:dart_arena_headless --config run.json', 'options': [ {'name': '--config', 'value': 'path', 'required': true}, + {'name': '--preset', 'value': 'mvp', 'required': false}, {'name': '--help', 'required': false}, ], 'configFormat': 'json', diff --git a/app/lib/core/task_bundle_digest.dart b/app/lib/core/task_bundle_digest.dart index 8ea77e6..9debaf6 100644 --- a/app/lib/core/task_bundle_digest.dart +++ b/app/lib/core/task_bundle_digest.dart @@ -16,6 +16,47 @@ const taskBundleDigestFileRoots = [ 'negative_cases', ]; +class CorpusManifestEntry { + const CorpusManifestEntry({ + required this.taskId, + required this.taskVersion, + required this.taskBundleDigest, + }); + + final String taskId; + final int taskVersion; + final String taskBundleDigest; + + Map toJson() => { + 'taskId': taskId, + 'taskVersion': taskVersion, + 'taskBundleDigest': taskBundleDigest, + }; +} + +String corpusManifestDigestSha256(Iterable entries) { + final sorted = entries.toList() + ..sort((a, b) { + final id = a.taskId.compareTo(b.taskId); + if (id != 0) return id; + final version = a.taskVersion.compareTo(b.taskVersion); + if (version != 0) return version; + return a.taskBundleDigest.compareTo(b.taskBundleDigest); + }); + return sha256 + .convert( + utf8.encode( + sorted + .map( + (entry) => + '${entry.taskId}\u0000${entry.taskVersion}\u0000${entry.taskBundleDigest}', + ) + .join('\n'), + ), + ) + .toString(); +} + Future taskBundleDigestSha256(Directory bundleDirectory) async { final files = await _taskBundleDigestFiles(bundleDirectory); final bytesBuilder = BytesBuilder(copy: false); diff --git a/app/lib/export/leaderboard_exporter.dart b/app/lib/export/leaderboard_exporter.dart index 19ad016..655edc6 100644 --- a/app/lib/export/leaderboard_exporter.dart +++ b/app/lib/export/leaderboard_exporter.dart @@ -84,6 +84,7 @@ Future> buildLeaderboardExport( for (final runId in sortedRunIds) if (runsById[runId] != null) runsById[runId]!, ]; + final corpusManifest = _corpusManifestForRun(runsById[selected.anchorRunId]); final taskKeys = selected.taskRuns.map(_taskKey).toSet(); final modelKeys = selected.taskRuns .map((taskRun) => '${taskRun.providerId}:${taskRun.modelId}') @@ -132,7 +133,7 @@ Future> buildLeaderboardExport( ); return { - 'schemaVersion': 1, + 'schemaVersion': 2, 'generatedAt': generatedAt.toIso8601String(), 'benchmark': { 'name': 'PickArena', @@ -143,6 +144,11 @@ Future> buildLeaderboardExport( 'evaluatorSchemaVersion': dartArenaEvaluatorSchemaVersion, 'track': options.track, 'dataPolicy': options.strategy.kebabName, + if (corpusManifest != null) ...{ + 'preset': corpusManifest['preset'], + 'selectedTasks': corpusManifest['tasks'], + 'corpusManifestDigestSha256': corpusManifest['digestSha256'], + }, }, 'source': { 'anchorRunId': selected.anchorRunId, @@ -167,6 +173,21 @@ Future> buildLeaderboardExport( }; } +Map? _corpusManifestForRun(Run? run) { + if (run == null) return null; + final provenance = _decodeRunProvenance(run.provenanceJson); + if (provenance == null) return null; + final manifest = _objectMap( + _objectMap(provenance['config'])['corpusManifest'], + ); + if (manifest['preset'] is! String || + manifest['tasks'] is! List || + manifest['digestSha256'] is! String) { + return null; + } + return manifest; +} + _SelectedTaskRuns _selectTaskRuns({ required LeaderboardExportOptions options, required List completedRuns, @@ -188,6 +209,7 @@ _SelectedTaskRuns _selectTaskRuns({ final anchorSignature = _RunCompatibilitySignature.fromTaskRuns( taskRunsByRunId[anchorRun.id] ?? const [], scoringSchemaVersion: anchorWeights.scoringSchemaVersion, + corpusManifestDigest: _corpusManifestDigestForRun(anchorRun), harnessKinds: _harnessKinds( anchorRun, taskRunsByRunId[anchorRun.id] ?? const [], @@ -202,6 +224,7 @@ _SelectedTaskRuns _selectTaskRuns({ final candidateSignature = _RunCompatibilitySignature.fromTaskRuns( entry.value, scoringSchemaVersion: candidateWeights.scoringSchemaVersion, + corpusManifestDigest: _corpusManifestDigestForRun(candidateRun), harnessKinds: _harnessKinds(candidateRun, entry.value), ); if (!anchorSignature.isCompatibleWith(candidateSignature)) continue; @@ -1260,11 +1283,13 @@ class _RunCompatibilitySignature { required this.harnessIds, required this.harnessKinds, required this.scoringSchemaVersion, + required this.corpusManifestDigest, }); factory _RunCompatibilitySignature.fromTaskRuns( List taskRuns, { required int? scoringSchemaVersion, + required String? corpusManifestDigest, required Map harnessKinds, }) { return _RunCompatibilitySignature( @@ -1283,6 +1308,7 @@ class _RunCompatibilitySignature { .toList() ..sort()), scoringSchemaVersion: scoringSchemaVersion, + corpusManifestDigest: corpusManifestDigest, ); } @@ -1290,11 +1316,13 @@ class _RunCompatibilitySignature { final List harnessIds; final List harnessKinds; final int? scoringSchemaVersion; + final String? corpusManifestDigest; bool isCompatibleWith(_RunCompatibilitySignature other) { return _listEquals(taskKeys, other.taskKeys) && _listEquals(harnessIds, other.harnessIds) && _listEquals(harnessKinds, other.harnessKinds) && + corpusManifestDigest == other.corpusManifestDigest && (scoringSchemaVersion == null || other.scoringSchemaVersion == null || scoringSchemaVersion == other.scoringSchemaVersion); @@ -1306,6 +1334,7 @@ class _RunCompatibilitySignature { _listEquals(taskKeys, other.taskKeys) && _listEquals(harnessIds, other.harnessIds) && _listEquals(harnessKinds, other.harnessKinds) && + corpusManifestDigest == other.corpusManifestDigest && scoringSchemaVersion == other.scoringSchemaVersion; } @@ -1315,9 +1344,15 @@ class _RunCompatibilitySignature { Object.hashAll(harnessIds), Object.hashAll(harnessKinds), scoringSchemaVersion, + corpusManifestDigest, ); } +String? _corpusManifestDigestForRun(Run run) { + final digest = _corpusManifestForRun(run)?['digestSha256']; + return digest is String && digest.isNotEmpty ? digest : null; +} + Map _harnessKinds(Run run, List taskRuns) { final provenanceJson = run.provenanceJson; if (provenanceJson == null || provenanceJson.trim().isEmpty) return const {}; diff --git a/app/lib/export/release_report.dart b/app/lib/export/release_report.dart index 7b70c3f..511abe4 100644 --- a/app/lib/export/release_report.dart +++ b/app/lib/export/release_report.dart @@ -14,7 +14,7 @@ const _requiredArtifactBundleChecksumPaths = [ ..._standardBundleFilePaths, ]; -const _artifactBundleManifestSchemaVersion = 1; +const _artifactBundleManifestSchemaVersions = {1, 2}; const _artifactBundleChecksumsSchemaVersion = 1; const _artifactRunResultsSchemaVersion = 1; const _taskQaSummarySchemaVersion = 1; @@ -202,6 +202,23 @@ Map buildReleaseReport({ if (taskSetId == null) { blockers.add('Leaderboard benchmark task set id metadata is missing.'); } + final preset = _nonEmptyString(benchmark['preset']); + final corpusManifestDigest = _nonEmptyString( + benchmark['corpusManifestDigestSha256'], + ); + final selectedTasks = benchmark['selectedTasks']; + final digestIsSha256 = + corpusManifestDigest != null && + RegExp(r'^[0-9a-f]{64}$').hasMatch(corpusManifestDigest); + if (preset == null || + !digestIsSha256 || + selectedTasks is! List || + selectedTasks.isEmpty) { + blockers.add( + 'Leaderboard is missing the frozen corpus manifest; run with --preset to ' + 'snapshot the corpus before an official release.', + ); + } if (evaluatorSchemaVersion <= 0) { blockers.add( 'Leaderboard benchmark evaluator schema version metadata is missing.', @@ -344,7 +361,7 @@ Map buildReleaseReport({ ); return { - 'schemaVersion': 1, + 'schemaVersion': 2, 'releaseId': options.releaseId, 'generatedAt': generatedAt.toIso8601String(), 'status': blockers.isEmpty ? 'ready' : 'blocked', @@ -362,6 +379,9 @@ Map buildReleaseReport({ 'evaluatorSchemaVersion': evaluatorSchemaVersion, 'track': benchmark['track'], 'dataPolicy': benchmark['dataPolicy'], + 'preset': benchmark['preset'], + 'selectedTasks': benchmark['selectedTasks'], + 'corpusManifestDigestSha256': benchmark['corpusManifestDigestSha256'], }, 'source': { 'anchorRunId': source['anchorRunId'], @@ -1749,7 +1769,7 @@ Map _artifactBundleSummary({ if (schemaVersion <= 0) { blockers.add('Run artifact bundle manifest schema version is missing.'); - } else if (schemaVersion != _artifactBundleManifestSchemaVersion) { + } else if (!_artifactBundleManifestSchemaVersions.contains(schemaVersion)) { blockers.add( 'Run artifact bundle manifest schema version $schemaVersion is unsupported.', ); @@ -2016,7 +2036,7 @@ Map _artifactBundleSummary({ final complete = schemaVersion > 0 && - schemaVersion == _artifactBundleManifestSchemaVersion && + _artifactBundleManifestSchemaVersions.contains(schemaVersion) && manifestRunId != null && runIdInLeaderboardSource && manifestTaskRunCount > 0 && @@ -6344,11 +6364,37 @@ Map _taskQaSummary( 'track': entry['track'], 'status': entry['status'], 'failureCount': _intValue(entry['failureCount']), + ..._f2pP2pForTaskQaReport(_taskQaReportForEntry(reports, entry)), }, ], }; } +Map? _taskQaReportForEntry( + List> reports, + Map entry, +) { + final key = _taskQaReportKey(entry); + if (key == null) return null; + for (final report in reports) { + if (_taskQaReportKey(report) == key) return report; + } + return null; +} + +Map _f2pP2pForTaskQaReport(Map? report) { + final checks = _objectMap(report?['checks']); + final f2p = checks['baselineHiddenFailed']; + final publicPassed = checks['referencePublicPassed']; + final hiddenPassed = checks['referenceHiddenPassed']; + return { + 'f2p': f2p is bool ? f2p : null, + 'p2p': publicPassed is bool && hiddenPassed is bool + ? publicPassed && hiddenPassed + : null, + }; +} + Map _taskQaSummaryIntegrityAudit({ required Map summary, required Object? rawReports, diff --git a/app/lib/export/run_manifest.dart b/app/lib/export/run_manifest.dart index 7ac9fb8..820903d 100644 --- a/app/lib/export/run_manifest.dart +++ b/app/lib/export/run_manifest.dart @@ -82,7 +82,7 @@ class RunManifestV1 { final Object? provenance; Map toJson() => { - 'schemaVersion': 1, + 'schemaVersion': 2, 'generatedAt': generatedAt, 'run': run, 'appVersion': appVersion, diff --git a/app/lib/headless/headless_benchmark_runner.dart b/app/lib/headless/headless_benchmark_runner.dart index 5bb1f25..ef8279d 100644 --- a/app/lib/headless/headless_benchmark_runner.dart +++ b/app/lib/headless/headless_benchmark_runner.dart @@ -30,6 +30,8 @@ class HeadlessBenchmarkConfig { required this.modelsByProvider, this.agentHarnesses = const [], this.agentHarnessProvenance = const {}, + this.preset, + this.corpusManifest, required this.evaluatorConfig, required this.evaluatorWeights, required this.workdirManager, @@ -60,6 +62,8 @@ class HeadlessBenchmarkConfig { final Map> modelsByProvider; final List agentHarnesses; final Map> agentHarnessProvenance; + final String? preset; + final Map? corpusManifest; final EvaluatorConfig evaluatorConfig; final Map evaluatorWeights; final WorkdirManager workdirManager; @@ -223,6 +227,8 @@ class HeadlessBenchmarkRunner { generatedCodeSandboxEnforced: config.generatedCodeSandboxEnforced, generatedCodeSandboxBackend: config.generatedCodeSandboxBackend, agentHarnessProvenance: config.agentHarnessProvenance, + preset: config.preset, + corpusManifest: config.corpusManifest, ); final provenanceJson = await buildRunProvenanceJson( runId: config.runId, diff --git a/app/lib/headless/headless_cli_config.dart b/app/lib/headless/headless_cli_config.dart index 59ec358..12b181c 100644 --- a/app/lib/headless/headless_cli_config.dart +++ b/app/lib/headless/headless_cli_config.dart @@ -27,6 +27,7 @@ class HeadlessCliConfig { required this.databasePath, required this.timeout, this.name, + this.preset, this.judge, this.taskBundleRoots = const [], this.maxConcurrency = 4, @@ -38,6 +39,7 @@ class HeadlessCliConfig { final String configPath; final String runId; final String? name; + final String? preset; final List tasks; final List providers; final HeadlessCliJudgeConfig? judge; @@ -88,7 +90,10 @@ class HeadlessCliJudgeConfig { final String model; } -Future loadHeadlessCliConfig(File file) async { +Future loadHeadlessCliConfig( + File file, { + String? presetOverride, +}) async { Object? decoded; try { decoded = jsonDecode(await file.readAsString()); @@ -100,6 +105,11 @@ Future loadHeadlessCliConfig(File file) async { if (decoded is! Map) { throw const HeadlessCliConfigException('config must be a JSON object'); } + if (presetOverride != null) { + decoded = Map.from(decoded) + ..remove('tasks') + ..['preset'] = presetOverride; + } return parseHeadlessCliConfig( decoded, configPath: p.normalize(p.absolute(file.path)), @@ -114,10 +124,25 @@ HeadlessCliConfig parseHeadlessCliConfig( final runId = _requiredString(json, 'runId'); _validateSafeSegment(runId, 'runId'); - final tasks = _requiredStringList(json, 'tasks'); - if (tasks.isEmpty) { + final hasTasks = json.containsKey('tasks'); + final tasks = hasTasks + ? _requiredStringList(json, 'tasks') + : const []; + final preset = _optionalString(json, 'preset'); + if (hasTasks && tasks.isEmpty) { throw const HeadlessCliConfigException('tasks must not be empty'); } + if (preset != null && preset.trim().isEmpty) { + throw const HeadlessCliConfigException('preset must not be empty'); + } + if ((!hasTasks && preset == null) || (hasTasks && preset != null)) { + throw const HeadlessCliConfigException( + 'exactly one of tasks or preset must be provided', + ); + } + if (preset != null && preset != 'mvp') { + throw HeadlessCliConfigException('unknown preset: $preset, expected: mvp'); + } final providersJson = _requiredList(json, 'providers'); if (providersJson.isEmpty) { @@ -169,6 +194,7 @@ HeadlessCliConfig parseHeadlessCliConfig( configPath: configPath, runId: runId, name: _optionalString(json, 'name'), + preset: preset, tasks: List.unmodifiable(tasks), providers: List.unmodifiable(providers), judge: judge, diff --git a/app/lib/headless/headless_cli_runner.dart b/app/lib/headless/headless_cli_runner.dart index 7b6a029..6e52e04 100644 --- a/app/lib/headless/headless_cli_runner.dart +++ b/app/lib/headless/headless_cli_runner.dart @@ -6,11 +6,13 @@ import 'package:dart_arena/agent/command_template_agent_harness.dart'; import 'package:dart_arena/agent/droid_agent_harness.dart'; import 'package:dart_arena/agent/minimal_agent_harness.dart'; import 'package:dart_arena/core/benchmark_task.dart'; +import 'package:dart_arena/core/task_bundle_digest.dart'; import 'package:dart_arena/core/evaluator_config.dart'; import 'package:dart_arena/core/task_registry.dart'; import 'package:dart_arena/core/model_response.dart'; import 'package:dart_arena/headless/headless_benchmark_runner.dart'; import 'package:dart_arena/headless/headless_cli_config.dart'; +import 'package:dart_arena/headless/task_preset.dart'; import 'package:dart_arena/providers/anthropic_provider.dart'; import 'package:dart_arena/providers/deepseek_provider.dart'; import 'package:dart_arena/providers/droid_exec_provider.dart'; @@ -88,21 +90,30 @@ Future runHeadlessCli( var runnerInvoked = false; try { - final configPath = _parseConfigArg(args); - if (configPath == null) { + final cliArgs = _parseConfigArg(args); + if (cliArgs == null) { out(jsonEncode(_helpJson())); return 0; } - final cliConfig = await loadHeadlessCliConfig(File(configPath)); + final cliConfig = await loadHeadlessCliConfig( + File(cliArgs.configPath), + presetOverride: cliArgs.preset, + ); final generatedCodeSandbox = await dependencies.generatedCodeSandboxBuilder( cliConfig, ); final registry = dependencies.taskRegistryBuilder(); await _registerFileBackedTasks(registry, cliConfig.taskBundleRoots); - final tasks = [ - for (final taskId in cliConfig.tasks) _resolveTask(registry, taskId), - ]; + final tasks = cliConfig.preset == null + ? [for (final taskId in cliConfig.tasks) _resolveTask(registry, taskId)] + : await selectTaskPreset(cliConfig.preset!, registry.all()); + if (tasks.isEmpty) { + throw HeadlessCliConfigException( + 'preset ${cliConfig.preset} selected no tasks', + ); + } + final corpusManifest = await _corpusManifest(cliConfig.preset, tasks); if (cliConfig.providers.any((provider) => provider.type == 'agent_cli') && tasks.any((task) => task.track == BenchmarkTrack.codegen)) { throw const HeadlessCliConfigException( @@ -146,6 +157,8 @@ Future runHeadlessCli( for (final harness in agentHarnesses) harness.id: _agentHarnessProvenance(harness), }, + preset: cliConfig.preset, + corpusManifest: corpusManifest, evaluatorConfig: evaluatorConfig, evaluatorWeights: cliConfig.evaluatorWeights, workdirManager: WorkdirManager( @@ -206,6 +219,34 @@ Future runHeadlessCli( } } +Future?> _corpusManifest( + String? preset, + List tasks, +) async { + if (preset == null) return null; + final entries = []; + for (final task in tasks) { + if (task is! FileBackedTask) { + throw HeadlessCliConfigException( + 'preset $preset requires file-backed tasks', + ); + } + entries.add( + CorpusManifestEntry( + taskId: task.id, + taskVersion: task.version, + taskBundleDigest: await taskBundleDigestSha256(task.bundleDirectory), + ), + ); + } + entries.sort((a, b) => a.taskId.compareTo(b.taskId)); + return { + 'preset': preset, + 'tasks': [for (final entry in entries) entry.toJson()], + 'digestSha256': corpusManifestDigestSha256(entries), + }; +} + Future _defaultGeneratedCodeSandboxBuilder( HeadlessCliConfig config, ) async { @@ -233,11 +274,12 @@ Future _registerFileBackedTasks( } } -String? _parseConfigArg(List args) { +_HeadlessCliArgs? _parseConfigArg(List args) { if (args.isEmpty || args.contains('--help') || args.contains('-h')) { return null; } String? configPath; + String? preset; for (var i = 0; i < args.length; i++) { final arg = args[i]; if (arg == '--config') { @@ -250,6 +292,16 @@ String? _parseConfigArg(List args) { ); } configPath = args[++i]; + } else if (arg == '--preset') { + if (i + 1 >= args.length) { + throw const HeadlessCliConfigException('--preset requires a name'); + } + if (preset != null) { + throw const HeadlessCliConfigException( + '--preset may only be provided once', + ); + } + preset = args[++i]; } else { throw HeadlessCliConfigException('unknown argument: $arg'); } @@ -257,7 +309,14 @@ String? _parseConfigArg(List args) { if (configPath == null) { throw const HeadlessCliConfigException('--config is required'); } - return configPath; + return _HeadlessCliArgs(configPath: configPath, preset: preset); +} + +class _HeadlessCliArgs { + const _HeadlessCliArgs({required this.configPath, this.preset}); + + final String configPath; + final String? preset; } Map _helpJson() { @@ -267,6 +326,7 @@ Map _helpJson() { 'dart run --verbosity=error dart_arena:dart_arena_headless --config run.json', 'options': [ {'name': '--config', 'value': 'path', 'required': true}, + {'name': '--preset', 'value': 'name', 'required': false}, {'name': '--help', 'required': false}, ], 'configFormat': 'json', diff --git a/app/lib/headless/task_preset.dart b/app/lib/headless/task_preset.dart new file mode 100644 index 0000000..cd6bb90 --- /dev/null +++ b/app/lib/headless/task_preset.dart @@ -0,0 +1,52 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:dart_arena/core/benchmark_task.dart'; +import 'package:dart_arena/core/task_bundle_digest.dart'; +import 'package:dart_arena/headless/headless_cli_config.dart'; +import 'package:dart_arena/tasks/file_backed/file_backed_task.dart'; + +Future> selectTaskPreset( + String preset, + Iterable tasks, +) async { + if (preset != 'mvp') { + throw HeadlessCliConfigException('unknown preset: $preset, expected: mvp'); + } + final selected = []; + for (final task in tasks) { + if (task is! FileBackedTask || + !task.isFlutter || + task.track != BenchmarkTrack.agentic || + task.releaseMetadata.corpus != TaskCorpus.privateOfficial || + task.releaseMetadata.status != TaskReleaseStatus.active || + !await _hasAdmittedQaReport(task)) { + continue; + } + selected.add(task); + } + selected.sort((a, b) => a.id.compareTo(b.id)); + return List.unmodifiable(selected); +} + +Future _hasAdmittedQaReport(FileBackedTask task) async { + final report = File( + '${task.bundleDirectory.path}${Platform.pathSeparator}qa${Platform.pathSeparator}admission_report.json', + ); + try { + final decoded = jsonDecode(await report.readAsString()); + if (decoded is! Map || + decoded['status'] != 'admitted' || + decoded['taskId'] != task.id || + decoded['taskVersion'] != task.version || + decoded['track'] != task.track.name) { + return false; + } + final admission = decoded['admission']; + if (admission is! Map) return false; + final expectedDigest = await taskBundleDigestSha256(task.bundleDirectory); + return admission['taskBundleDigest'] == expectedDigest; + } on Object { + return false; + } +} diff --git a/app/lib/runner/run_provenance.dart b/app/lib/runner/run_provenance.dart index 3ecdcc8..00dabd5 100644 --- a/app/lib/runner/run_provenance.dart +++ b/app/lib/runner/run_provenance.dart @@ -50,6 +50,8 @@ class RunProvenanceConfig { this.generatedCodeSandboxEnforced = false, this.generatedCodeSandboxBackend, this.agentHarnessProvenance = const {}, + this.preset, + this.corpusManifest, }); final List tasks; @@ -63,6 +65,8 @@ class RunProvenanceConfig { final bool generatedCodeSandboxEnforced; final String? generatedCodeSandboxBackend; final Map> agentHarnessProvenance; + final String? preset; + final Map? corpusManifest; } class DefaultRunProvenanceEnvironmentProvider @@ -247,7 +251,7 @@ Future buildRunProvenanceJson({ }; final json = { - 'schemaVersion': 1, + 'schemaVersion': 2, 'capturedAt': capturedAt.toUtc().toIso8601String(), 'runId': runId, 'config': { @@ -263,6 +267,9 @@ Future buildRunProvenanceJson({ 'backend': config.generatedCodeSandboxBackend, }, 'agentHarnesses': config.agentHarnessProvenance, + if (config.preset != null) 'preset': config.preset, + if (config.corpusManifest != null) + 'corpusManifest': config.corpusManifest, 'pricingRegistry': pricingRegistryProvenance(), 'modelsByProvider': _sortedModelsMap(normalizedModelsByProvider), 'evaluatorWeights': _sortedNumberMap(evaluatorWeights), diff --git a/app/test/core/task_bundle_digest_test.dart b/app/test/core/task_bundle_digest_test.dart index 487f9de..8e599a4 100644 --- a/app/test/core/task_bundle_digest_test.dart +++ b/app/test/core/task_bundle_digest_test.dart @@ -8,6 +8,51 @@ import 'package:path/path.dart' as p; import 'package:test/test.dart'; void main() { + group('corpus manifest digest', () { + test('is deterministic and changes with task version or bundle digest', () { + final a = List.filled(64, 'a').join(); + final b = List.filled(64, 'b').join(); + final c = List.filled(64, 'c').join(); + final first = [ + CorpusManifestEntry( + taskId: 'task.b', + taskVersion: 1, + taskBundleDigest: b, + ), + CorpusManifestEntry( + taskId: 'task.a', + taskVersion: 1, + taskBundleDigest: a, + ), + ]; + final digest = corpusManifestDigestSha256(first); + + expect(corpusManifestDigestSha256(first.reversed), digest); + expect( + corpusManifestDigestSha256([ + CorpusManifestEntry( + taskId: 'task.a', + taskVersion: 2, + taskBundleDigest: a, + ), + first.first, + ]), + isNot(digest), + ); + expect( + corpusManifestDigestSha256([ + CorpusManifestEntry( + taskId: 'task.a', + taskVersion: 1, + taskBundleDigest: c, + ), + first.first, + ]), + isNot(digest), + ); + }); + }); + test('task bundle digest follows declared files and excludes qa', () async { final root = await Directory.systemTemp.createTemp('task_bundle_digest_'); addTearDown(() async { diff --git a/app/test/export/artifact_bundle_test.dart b/app/test/export/artifact_bundle_test.dart index a5c49e9..5d1df71 100644 --- a/app/test/export/artifact_bundle_test.dart +++ b/app/test/export/artifact_bundle_test.dart @@ -162,7 +162,7 @@ void main() { File(p.join(target.path, 'manifest.json')).readAsStringSync(), ) as Map; - expect(manifest['schemaVersion'], 1); + expect(manifest['schemaVersion'], 2); expect(manifest['appVersion'], '1.0.0+test'); expect(manifest['driftSchemaVersion'], 7); expect((manifest['provenance'] as Map)['runId'], 'r1'); diff --git a/app/test/export/leaderboard_cli_runner_test.dart b/app/test/export/leaderboard_cli_runner_test.dart index 13a1e1a..808f0ff 100644 --- a/app/test/export/leaderboard_cli_runner_test.dart +++ b/app/test/export/leaderboard_cli_runner_test.dart @@ -140,7 +140,7 @@ void main() { expect(exitCode, 0); expect(stdoutLines.single, contains('"status":"completed"')); final text = File(outPath).readAsStringSync(); - expect(text, startsWith('{\n "schemaVersion": 1,')); + expect(text, startsWith('{\n "schemaVersion": 2,')); final decoded = jsonDecode(text) as Map; expect(decoded['generatedAt'], '2026-05-31T00:00:00.000Z'); expect((decoded['models']! as List), hasLength(1)); diff --git a/app/test/export/leaderboard_exporter_test.dart b/app/test/export/leaderboard_exporter_test.dart index 900b71d..51be174 100644 --- a/app/test/export/leaderboard_exporter_test.dart +++ b/app/test/export/leaderboard_exporter_test.dart @@ -22,6 +22,7 @@ void main() { db, id: 'compatible-old', completedAt: DateTime.utc(2026, 5, 1), + provenanceJson: _presetProvenanceJson(), ); await _seedTaskRun( db, @@ -42,6 +43,7 @@ void main() { db, id: 'compatible-latest', completedAt: DateTime.utc(2026, 5, 2), + provenanceJson: _presetProvenanceJson(), ); await _seedTaskRun( db, @@ -100,6 +102,9 @@ void main() { expect(benchmark['version'], '2026-05-31-master-spec'); expect(benchmark['taskSetId'], startsWith('taskset-')); expect(benchmark['evaluatorSchemaVersion'], 2); + expect(benchmark['preset'], 'mvp'); + expect(benchmark['corpusManifestDigestSha256'], hasLength(64)); + expect(benchmark['selectedTasks'], hasLength(2)); final pricingRegistry = export['pricingRegistry']! as Map; expect(pricingRegistry['version'], '2026-05-31'); @@ -166,6 +171,42 @@ void main() { }, ); + test('aggregate-compatible separates corpus manifest digests', () async { + await _seedRun( + db, + id: 'manifest-old', + completedAt: DateTime.utc(2026, 5, 1), + provenanceJson: _presetProvenanceJson(corpusDigest: 'a'), + ); + await _seedTaskRun( + db, + id: 'manifest-old-a', + runId: 'manifest-old', + taskId: 'task.a', + ); + await _seedRun( + db, + id: 'manifest-latest', + completedAt: DateTime.utc(2026, 5, 2), + provenanceJson: _presetProvenanceJson(corpusDigest: 'b'), + ); + await _seedTaskRun( + db, + id: 'manifest-latest-a', + runId: 'manifest-latest', + taskId: 'task.a', + ); + + final export = await buildLeaderboardExport( + db, + options: const LeaderboardExportOptions(track: 'agentic'), + ); + + expect((export['source']! as Map)['runIds'], [ + 'manifest-latest', + ]); + }); + test('aggregate-compatible separates command-template versions', () async { for (final entry in [ ('old', DateTime.utc(2026, 5, 1), '1.0.0'), @@ -1371,6 +1412,30 @@ String _harnessProvenanceJson(String kind, {String? agent, String? version}) => }, }); +String _presetProvenanceJson({String corpusDigest = 'c'}) => jsonEncode({ + 'schemaVersion': 2, + 'config': { + 'scoringSchemaVersion': 2, + 'evaluatorWeights': {'compile': 1.0}, + 'corpusManifest': { + 'preset': 'mvp', + 'tasks': [ + { + 'taskId': 'task.a', + 'taskVersion': 1, + 'taskBundleDigest': List.filled(64, 'a').join(), + }, + { + 'taskId': 'task.b', + 'taskVersion': 1, + 'taskBundleDigest': List.filled(64, 'b').join(), + }, + ], + 'digestSha256': List.filled(64, corpusDigest).join(), + }, + }, +}); + Future _seedTaskRun( AppDatabase db, { required String id, diff --git a/app/test/export/release_report_cli_runner_test.dart b/app/test/export/release_report_cli_runner_test.dart index f4ece98..14f3205 100644 --- a/app/test/export/release_report_cli_runner_test.dart +++ b/app/test/export/release_report_cli_runner_test.dart @@ -7045,10 +7045,6 @@ void main() { final report = jsonDecode(await File(outPath).readAsString()) as Map; final blockers = (report['blockers']! as List).join('\n'); - expect( - blockers, - contains('Run artifact bundle manifest schema version 2 is unsupported.'), - ); expect( blockers, contains( @@ -10208,6 +10204,13 @@ void main() { expect(benchmark['version'], null); expect(benchmark['taskSetId'], null); expect(benchmark['evaluatorSchemaVersion'], 0); + expect( + blockers, + contains( + 'Leaderboard is missing the frozen corpus manifest; run with --preset ' + 'to snapshot the corpus before an official release.', + ), + ); }); test('blocks report when benchmark evaluator schema is stale', () async { @@ -10272,6 +10275,67 @@ void main() { expect(reportBenchmark['evaluatorSchemaVersion'], 1); }); + test('blocks report when corpus manifest digest is not a sha256', () async { + final tmp = await Directory.systemTemp.createTemp( + 'release_report_corpus_digest_', + ); + addTearDown(() async { + if (await tmp.exists()) await tmp.delete(recursive: true); + }); + final leaderboardPath = p.join(tmp.path, 'leaderboard.v1.json'); + final taskQaDir = Directory(p.join(tmp.path, 'task_qa')); + await taskQaDir.create(recursive: true); + final taskQaSummaryPath = p.join(taskQaDir.path, 'admission_summary.json'); + final reportPath = p.join(taskQaDir.path, 'tasks', 'a', 'report.json'); + await Directory(p.dirname(reportPath)).create(recursive: true); + final leaderboard = _leaderboardJson(sampleCount: 2); + final benchmark = leaderboard['benchmark']! as Map; + benchmark['corpusManifestDigestSha256'] = 'not-a-sha256'; + await File(leaderboardPath).writeAsString(_prettyJson(leaderboard)); + await File( + taskQaSummaryPath, + ).writeAsString(_prettyJson(_taskQaSummaryJson(reportPath: reportPath))); + await File(reportPath).writeAsString(_prettyJson(_taskQaReportJson())); + final databasePath = p.join(tmp.path, 'dart_arena.sqlite'); + await _seedDatabase(databasePath); + final outPath = p.join(tmp.path, 'release_report.v1.json'); + final stderrLines = []; + + final exitCode = await runReleaseReportCli( + [ + '--leaderboard', + leaderboardPath, + '--task-qa-summary', + taskQaSummaryPath, + '--database', + databasePath, + '--out', + outPath, + '--release-id', + '2026-06-corpus-digest-gated', + '--fail-on-blocked', + ], + dependencies: ReleaseReportCliDependencies( + now: () => DateTime.utc(2026, 6, 4, 2, 2), + ), + stdoutWriter: (_) {}, + stderrWriter: stderrLines.add, + ); + + expect(exitCode, 1); + expect(stderrLines.single, contains('"status":"blocked"')); + final report = + jsonDecode(await File(outPath).readAsString()) as Map; + final blockers = (report['blockers']! as List).join('\n'); + expect( + blockers, + contains( + 'Leaderboard is missing the frozen corpus manifest; run with --preset ' + 'to snapshot the corpus before an official release.', + ), + ); + }); + test('blocks report when scoring metadata is missing', () async { final tmp = await Directory.systemTemp.createTemp( 'release_report_scoring_', @@ -10931,6 +10995,10 @@ Map _leaderboardJson({ 'version': '2026-05-31-master-spec', 'taskSetId': 'taskset-test', 'evaluatorSchemaVersion': 2, + 'preset': 'mvp', + 'selectedTasks': ['task.a'], + 'corpusManifestDigestSha256': + '0000000000000000000000000000000000000000000000000000000000000000', }, 'track': 'agentic', 'dataPolicy': 'aggregate-compatible', diff --git a/app/test/headless/headless_cli_config_test.dart b/app/test/headless/headless_cli_config_test.dart index 6e5e272..34e2ea2 100644 --- a/app/test/headless/headless_cli_config_test.dart +++ b/app/test/headless/headless_cli_config_test.dart @@ -1,3 +1,4 @@ +import 'dart:convert'; import 'dart:io'; import 'package:dart_arena/core/scoring.dart'; @@ -81,6 +82,82 @@ void main() { ]); }); + test('accepts a preset instead of explicit tasks', () { + final config = _validConfig() + ..remove('tasks') + ..['preset'] = 'mvp'; + + expect( + parseHeadlessCliConfig( + config, + configPath: p.join(Directory.current.path, 'run.json'), + ).preset, + 'mvp', + ); + }); + + test('preset override replaces configured tasks', () async { + final tmp = await Directory.systemTemp.createTemp('dart_arena_preset_'); + addTearDown(() => tmp.delete(recursive: true)); + final file = File(p.join(tmp.path, 'run.json')); + await file.writeAsString(jsonEncode(_validConfig())); + + final config = await loadHeadlessCliConfig(file, presetOverride: 'mvp'); + + expect(config.preset, 'mvp'); + expect(config.tasks, isEmpty); + }); + + test('rejects a preset with explicit tasks', () { + expect( + () => parseHeadlessCliConfig({ + ..._validConfig(), + 'preset': 'mvp', + }, configPath: p.join(Directory.current.path, 'run.json')), + throwsA(isA()), + ); + }); + + test('rejects a config without tasks or a preset', () { + final config = _validConfig()..remove('tasks'); + + expect( + () => parseHeadlessCliConfig( + config, + configPath: p.join(Directory.current.path, 'run.json'), + ), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('exactly one of tasks or preset must be provided'), + ), + ), + ); + }); + + test('rejects blank and unknown presets with config errors', () { + for (final preset in ['', ' ', 'bogus']) { + final config = _validConfig() + ..remove('tasks') + ..['preset'] = preset; + + expect( + () => parseHeadlessCliConfig( + config, + configPath: p.join(Directory.current.path, 'run.json'), + ), + throwsA( + isA().having( + (error) => error.message, + 'message', + isNot(isEmpty), + ), + ), + ); + } + }); + test('parses generated-code sandbox requirement flag', () { final config = parseHeadlessCliConfig({ ..._validConfig(), diff --git a/app/test/headless/task_preset_test.dart b/app/test/headless/task_preset_test.dart new file mode 100644 index 0000000..0b0805a --- /dev/null +++ b/app/test/headless/task_preset_test.dart @@ -0,0 +1,107 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:dart_arena/core/benchmark_task.dart'; +import 'package:dart_arena/headless/task_preset.dart'; +import 'package:dart_arena/tasks/file_backed/file_backed_task.dart'; +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; + +void main() { + test('mvp selects the admitted official Flutter task metadata set', () async { + final root = Directory( + p.join(Directory.current.path, '..', 'tasks', 'flutter'), + ); + final tasks = await loadFileBackedTasks(root); + + final selected = await selectTaskPreset('mvp', tasks); + + expect(selected, hasLength(10)); + expect( + selected.every( + (task) => + task.isFlutter && + task.track == BenchmarkTrack.agentic && + task.releaseMetadata.corpus == TaskCorpus.privateOfficial && + task.releaseMetadata.status == TaskReleaseStatus.active, + ), + isTrue, + ); + }); + + test('mvp excludes metadata-matching tasks without admitted QA', () async { + final root = Directory( + p.join(Directory.current.path, '..', 'tasks', 'flutter'), + ); + final source = (await loadFileBackedTasks(root)).first; + final tmp = await Directory.systemTemp.createTemp('dart_arena_preset_'); + addTearDown(() => tmp.delete(recursive: true)); + final bundle = Directory(p.join(tmp.path, 'task')); + await _copyDirectory(source.bundleDirectory, bundle); + await File(p.join(bundle.path, 'qa', 'admission_report.json')).delete(); + + final task = await FileBackedTask.load(bundle); + + expect(await selectTaskPreset('mvp', [task]), isEmpty); + }); + + test( + 'mvp excludes tasks whose admitted QA report identity mismatches', + () async { + final root = Directory( + p.join(Directory.current.path, '..', 'tasks', 'flutter'), + ); + final source = (await loadFileBackedTasks(root)).first; + final tmp = await Directory.systemTemp.createTemp('dart_arena_preset_'); + addTearDown(() => tmp.delete(recursive: true)); + final bundle = Directory(p.join(tmp.path, 'task')); + await _copyDirectory(source.bundleDirectory, bundle); + final reportFile = File( + p.join(bundle.path, 'qa', 'admission_report.json'), + ); + final report = + jsonDecode(await reportFile.readAsString()) as Map; + report['taskId'] = 'some.other_task'; + await reportFile.writeAsString(jsonEncode(report)); + + final task = await FileBackedTask.load(bundle); + + expect(await selectTaskPreset('mvp', [task]), isEmpty); + }, + ); + + test( + 'mvp excludes tasks whose bundle drifted from the admitted digest', + () async { + final root = Directory( + p.join(Directory.current.path, '..', 'tasks', 'flutter'), + ); + final source = (await loadFileBackedTasks(root)).first; + final tmp = await Directory.systemTemp.createTemp('dart_arena_preset_'); + addTearDown(() => tmp.delete(recursive: true)); + final bundle = Directory(p.join(tmp.path, 'task')); + await _copyDirectory(source.bundleDirectory, bundle); + final instruction = File(p.join(bundle.path, 'instruction.md')); + await instruction.writeAsString( + '${await instruction.readAsString()}\n\n', + ); + + final task = await FileBackedTask.load(bundle); + + expect(await selectTaskPreset('mvp', [task]), isEmpty); + }, + ); +} + +Future _copyDirectory(Directory source, Directory destination) async { + await for (final entity in source.list(recursive: true)) { + final relative = p.relative(entity.path, from: source.path); + final target = p.join(destination.path, relative); + if (entity is Directory) { + await Directory(target).create(recursive: true); + } else if (entity is File) { + await File(target).parent.create(recursive: true); + await entity.copy(target); + } + } +} diff --git a/app/test/runner/run_provenance_test.dart b/app/test/runner/run_provenance_test.dart index 044eb67..2dfa520 100644 --- a/app/test/runner/run_provenance_test.dart +++ b/app/test/runner/run_provenance_test.dart @@ -192,7 +192,7 @@ void main() { ); final decoded = jsonDecode(jsonText) as Map; - expect(decoded['schemaVersion'], 1); + expect(decoded['schemaVersion'], 2); expect(decoded['runId'], 'run-1'); final config = decoded['config'] as Map; diff --git a/scripts/run-official-bubblewrap-benchmark.sh b/scripts/run-official-bubblewrap-benchmark.sh index 3eba091..9b5cbf9 100755 --- a/scripts/run-official-bubblewrap-benchmark.sh +++ b/scripts/run-official-bubblewrap-benchmark.sh @@ -68,18 +68,7 @@ cat >"$run_json" < 2) return null; if (!isRecord(value.benchmark)) return null; if (!isRecord(value.source)) return null; if (!Array.isArray(value.models)) return null; @@ -340,7 +341,7 @@ function parseLeaderboard(value: unknown): LeaderboardData | null { : []; return { - schemaVersion: 1, + schemaVersion, provisional: value.provisional === true, generatedAt: typeof value.generatedAt === 'string' ? value.generatedAt : null, benchmark: {