Skip to content

Commit a69fa54

Browse files
ErichDonGublerErichDonGubler
authored
Merge pull request #61 from ErichDonGubler/triage-by-intermittency
Split intermittent outcomes into separate `triage` line items
2 parents dac8b9f + f2efcf0 commit a69fa54

2 files changed

Lines changed: 228 additions & 87 deletions

File tree

‎moz-webgpu-cts/src/main.rs‎

Lines changed: 228 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ use self::{
1717

1818
use std::{
1919
collections::{BTreeMap, BTreeSet},
20-
fmt::{Debug, Display},
20+
fmt::{self, Debug, Display, Formatter},
2121
fs,
2222
hash::Hash,
2323
io::{self, BufReader, BufWriter},
@@ -753,8 +753,55 @@ fn run(cli: Cli) -> ExitCode {
753753
"from metadata files, analyzing results…"
754754
));
755755

756-
type TestSet = BTreeSet<Arc<SectionHeader>>;
757-
type SubtestByTestSet = BTreeMap<Arc<SectionHeader>, IndexSet<Arc<SectionHeader>>>;
756+
#[derive(Clone, Default)]
757+
struct PermaAndIntermittent<T> {
758+
perma: T,
759+
intermittent: T,
760+
}
761+
762+
impl<T> Debug for PermaAndIntermittent<T>
763+
where
764+
T: Debug,
765+
{
766+
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
767+
let Self {
768+
perma,
769+
intermittent,
770+
} = self;
771+
f.debug_struct("") // the name is distracting, blank it out plz
772+
.field("perma", perma)
773+
.field("intermittent", intermittent)
774+
.finish()
775+
}
776+
}
777+
778+
impl<T> PermaAndIntermittent<T> {
779+
pub fn as_ref(&self) -> PermaAndIntermittent<&T> {
780+
let Self {
781+
perma,
782+
intermittent,
783+
} = self;
784+
PermaAndIntermittent {
785+
perma,
786+
intermittent,
787+
}
788+
}
789+
790+
pub fn map<U>(self, f: impl Fn(T) -> U) -> PermaAndIntermittent<U> {
791+
let Self {
792+
perma,
793+
intermittent,
794+
} = self;
795+
PermaAndIntermittent {
796+
perma: f(perma),
797+
intermittent: f(intermittent),
798+
}
799+
}
800+
}
801+
802+
type TestSet = PermaAndIntermittent<BTreeSet<Arc<SectionHeader>>>;
803+
type SubtestByTestSet =
804+
PermaAndIntermittent<BTreeMap<Arc<SectionHeader>, IndexSet<Arc<SectionHeader>>>>;
758805

759806
#[derive(Clone, Debug, Default)]
760807
struct PerPlatformAnalysis {
@@ -844,57 +891,101 @@ fn run(cli: Cli) -> ExitCode {
844891
analysis.for_each_platform_mut(|analysis| {
845892
analysis
846893
.tests_with_disabled_or_skip
894+
.perma
847895
.insert(test_name.clone());
848896
})
849897
}
850898

851-
fn apply_expectation<Out, F>(expectation: Expectation<Out>, f: F)
852-
where
853-
F: FnMut(Out),
854-
Out: EnumSetType,
899+
fn insert_in_test_set<Out>(
900+
poi: &mut TestSet,
901+
test_name: &Arc<SectionHeader>,
902+
expectation: Expectation<Out>,
903+
outcome: Out,
904+
) where
905+
Out: Debug + Default + EnumSetType,
906+
{
907+
if expectation.is_superset(&Expectation::permanent(outcome)) {
908+
if expectation.is_permanent() {
909+
&mut poi.perma
910+
} else {
911+
&mut poi.intermittent
912+
}
913+
.insert(test_name.clone());
914+
}
915+
}
916+
917+
fn insert_in_subtest_by_test_set<Out>(
918+
poi: &mut SubtestByTestSet,
919+
test_name: &Arc<SectionHeader>,
920+
subtest_name: &Arc<SectionHeader>,
921+
expectation: Expectation<Out>,
922+
outcome: Out,
923+
) where
924+
Out: Debug + Default + EnumSetType,
855925
{
856-
expectation.into_iter().for_each(f)
926+
if expectation.is_superset(&Expectation::permanent(outcome)) {
927+
if expectation.is_permanent() {
928+
&mut poi.perma
929+
} else {
930+
&mut poi.intermittent
931+
}
932+
.entry(test_name.clone())
933+
.or_default()
934+
.insert(subtest_name.clone());
935+
}
857936
}
937+
858938
if let Some(expectations) = expectations {
859939
fn analyze_test_outcome<F>(
860940
test_name: &Arc<SectionHeader>,
861-
outcome: TestOutcome,
941+
expectation: Expectation<TestOutcome>,
862942
mut receiver: F,
863943
) where
864944
F: FnMut(&mut dyn FnMut(&mut PerPlatformAnalysis)),
865945
{
866-
match outcome {
867-
TestOutcome::Ok => (),
868-
// We skip this because this test _should_ contain subtests with
869-
// `TIMEOUT` and `NOTRUN`, so we shouldn't actually miss anything.
870-
TestOutcome::Timeout => (),
871-
TestOutcome::Crash => receiver(&mut |analysis| {
872-
analysis.tests_with_crashes.insert(test_name.clone());
873-
}),
874-
TestOutcome::Error => receiver(&mut |analysis| {
875-
analysis.tests_with_runner_errors.insert(test_name.clone());
876-
}),
877-
TestOutcome::Skip => receiver(&mut |analysis| {
878-
analysis
879-
.tests_with_disabled_or_skip
880-
.insert(test_name.clone());
881-
}),
946+
for outcome in expectation.iter() {
947+
match outcome {
948+
TestOutcome::Ok => (),
949+
// We skip this because this test _should_ contain subtests with
950+
// `TIMEOUT` and `NOTRUN`, so we shouldn't actually miss anything.
951+
TestOutcome::Timeout => (),
952+
TestOutcome::Crash => receiver(&mut |analysis| {
953+
insert_in_test_set(
954+
&mut analysis.tests_with_crashes,
955+
test_name,
956+
expectation,
957+
outcome,
958+
)
959+
}),
960+
TestOutcome::Error => receiver(&mut |analysis| {
961+
insert_in_test_set(
962+
&mut analysis.tests_with_runner_errors,
963+
test_name,
964+
expectation,
965+
outcome,
966+
)
967+
}),
968+
TestOutcome::Skip => receiver(&mut |analysis| {
969+
insert_in_test_set(
970+
&mut analysis.tests_with_disabled_or_skip,
971+
test_name,
972+
expectation,
973+
outcome,
974+
)
975+
}),
976+
}
882977
}
883978
}
884979

885980
let apply_to_all_platforms = |analysis: &mut Analysis, expectation| {
886-
apply_expectation(expectation, |outcome| {
887-
analyze_test_outcome(&test_name, outcome, |f| {
888-
analysis.for_each_platform_mut(f)
889-
})
981+
analyze_test_outcome(&test_name, expectation, |f| {
982+
analysis.for_each_platform_mut(f)
890983
})
891984
};
892985
let apply_to_specific_platforms =
893986
|analysis: &mut Analysis, platform, expectation| {
894-
apply_expectation(expectation, |outcome| {
895-
analyze_test_outcome(&test_name, outcome, |f| {
896-
analysis.for_platform_mut(platform, f)
897-
})
987+
analyze_test_outcome(&test_name, expectation, |f| {
988+
analysis.for_platform_mut(platform, f)
898989
})
899990
};
900991

@@ -944,59 +1035,67 @@ fn run(cli: Cli) -> ExitCode {
9441035
analysis
9451036
.windows
9461037
.tests_with_disabled_or_skip
1038+
.perma
9471039
.insert(test_name.clone());
9481040
}
9491041

9501042
if let Some(expectations) = expectations {
9511043
fn analyze_subtest_outcome<Fo>(
9521044
test_name: &Arc<SectionHeader>,
9531045
subtest_name: &Arc<SectionHeader>,
954-
outcome: SubtestOutcome,
1046+
expectation: Expectation<SubtestOutcome>,
9551047
mut receiver: Fo,
9561048
) where
9571049
Fo: FnMut(&mut dyn FnMut(&mut PerPlatformAnalysis)),
9581050
{
959-
match outcome {
960-
SubtestOutcome::Pass => (),
961-
SubtestOutcome::Timeout | SubtestOutcome::NotRun => {
962-
receiver(&mut |analysis| {
963-
analysis
964-
.subtests_with_timeouts_by_test
965-
.entry(test_name.clone())
966-
.or_default()
967-
.insert(subtest_name.clone());
968-
})
1051+
for outcome in expectation.iter() {
1052+
match outcome {
1053+
SubtestOutcome::Pass => (),
1054+
SubtestOutcome::Timeout | SubtestOutcome::NotRun => {
1055+
receiver(&mut |analysis| {
1056+
insert_in_subtest_by_test_set(
1057+
&mut analysis.subtests_with_timeouts_by_test,
1058+
test_name,
1059+
subtest_name,
1060+
expectation,
1061+
outcome,
1062+
)
1063+
})
1064+
}
1065+
SubtestOutcome::Crash => receiver(&mut |analysis| {
1066+
insert_in_test_set(
1067+
&mut analysis.tests_with_crashes,
1068+
test_name,
1069+
expectation,
1070+
outcome,
1071+
)
1072+
}),
1073+
SubtestOutcome::Fail => receiver(&mut |analysis| {
1074+
insert_in_subtest_by_test_set(
1075+
&mut analysis.subtests_with_failures_by_test,
1076+
test_name,
1077+
subtest_name,
1078+
expectation,
1079+
outcome,
1080+
)
1081+
}),
9691082
}
970-
SubtestOutcome::Crash => receiver(&mut |analysis| {
971-
analysis.tests_with_crashes.insert(test_name.clone());
972-
}),
973-
SubtestOutcome::Fail => receiver(&mut |analysis| {
974-
analysis
975-
.subtests_with_failures_by_test
976-
.entry(test_name.clone())
977-
.or_default()
978-
.insert(subtest_name.clone());
979-
}),
9801083
}
9811084
}
9821085

9831086
let apply_to_all_platforms = |analysis: &mut Analysis, expectation| {
984-
apply_expectation(expectation, |outcome| {
985-
analyze_subtest_outcome(&test_name, &subtest_name, outcome, |f| {
986-
analysis.for_each_platform_mut(f)
987-
})
1087+
analyze_subtest_outcome(&test_name, &subtest_name, expectation, |f| {
1088+
analysis.for_each_platform_mut(f)
9881089
})
9891090
};
9901091
let apply_to_specific_platforms =
9911092
|analysis: &mut Analysis, platform, expectation| {
992-
apply_expectation(expectation, |outcome| {
993-
analyze_subtest_outcome(
994-
&test_name,
995-
&subtest_name,
996-
outcome,
997-
|f| analysis.for_platform_mut(platform, f),
998-
)
999-
})
1093+
analyze_subtest_outcome(
1094+
&test_name,
1095+
&subtest_name,
1096+
expectation,
1097+
|f| analysis.for_platform_mut(platform, f),
1098+
)
10001099
};
10011100

10021101
match expectations.into_inner() {
@@ -1047,29 +1146,75 @@ fn run(cli: Cli) -> ExitCode {
10471146
subtests_with_timeouts_by_test,
10481147
} = analysis;
10491148

1050-
let num_tests_with_runner_errors = tests_with_runner_errors.len();
1051-
let num_tests_with_disabled = tests_with_disabled_or_skip.len();
1052-
let num_tests_with_crashes = tests_with_crashes.len();
1053-
let num_tests_with_failures_somewhere = subtests_with_failures_by_test.len();
1054-
let num_subtests_with_failures_somewhere = subtests_with_failures_by_test
1055-
.iter()
1056-
.flat_map(|(_name, subtests)| subtests.iter())
1057-
.count();
1058-
let num_tests_with_timeouts_somewhere = subtests_with_timeouts_by_test.len();
1059-
let num_subtests_with_timeouts_somewhere = subtests_with_timeouts_by_test
1060-
.iter()
1061-
.flat_map(|(_name, subtests)| subtests.iter())
1062-
.count();
1149+
let PermaAndIntermittent {
1150+
perma: num_tests_with_perma_runner_errors,
1151+
intermittent: num_tests_with_intermittent_runner_errors,
1152+
} = tests_with_runner_errors.as_ref().map(|tests| tests.len());
1153+
1154+
let PermaAndIntermittent {
1155+
perma: num_tests_with_disabled,
1156+
intermittent: num_tests_with_intermittent_disabled,
1157+
} = tests_with_disabled_or_skip
1158+
.as_ref()
1159+
.map(|tests| tests.len());
1160+
let PermaAndIntermittent {
1161+
perma: num_tests_with_perma_crashes,
1162+
intermittent: num_tests_with_intermittent_crashes,
1163+
} = tests_with_crashes.as_ref().map(|tests| tests.len());
1164+
let PermaAndIntermittent {
1165+
perma: num_tests_with_perma_failures_somewhere,
1166+
intermittent: num_tests_with_intermittent_failures_somewhere,
1167+
} = subtests_with_failures_by_test
1168+
.as_ref()
1169+
.map(|tests| tests.len());
1170+
let PermaAndIntermittent {
1171+
perma: num_subtests_with_perma_failures_somewhere,
1172+
intermittent: num_subtests_with_intermittent_failures_somewhere,
1173+
} = subtests_with_failures_by_test.as_ref().map(|tests| {
1174+
tests
1175+
.iter()
1176+
.flat_map(|(_name, subtests)| subtests.iter())
1177+
.count()
1178+
});
1179+
let PermaAndIntermittent {
1180+
perma: num_tests_with_perma_timeouts_somewhere,
1181+
intermittent: num_tests_with_intermittent_timeouts_somewhere,
1182+
} = subtests_with_timeouts_by_test
1183+
.as_ref()
1184+
.map(|tests| tests.len());
1185+
let PermaAndIntermittent {
1186+
perma: num_subtests_with_perma_timeouts_somewhere,
1187+
intermittent: num_subtests_with_intermittent_timeouts_somewhere,
1188+
} = subtests_with_timeouts_by_test.as_ref().map(|tests| {
1189+
tests
1190+
.iter()
1191+
.flat_map(|(_name, subtests)| subtests.iter())
1192+
.count()
1193+
});
1194+
1195+
if num_tests_with_intermittent_disabled > 0 {
1196+
log::warn!(
1197+
concat!("found {} intermittent `SKIP` outcomes, which we don't understand yet; figure it out! The tests: {:#?}"),
1198+
num_tests_with_intermittent_disabled,
1199+
tests_with_disabled_or_skip,
1200+
)
1201+
}
1202+
10631203
println!(
10641204
"\
10651205
{platform:?}:
10661206
HIGH PRIORITY:
1067-
{num_tests_with_runner_errors} test(s) with execution reporting `ERROR`
1207+
{num_tests_with_perma_runner_errors} test(s) with execution reporting permanent `ERROR`
10681208
{num_tests_with_disabled} test(s) with some portion marked as `disabled`
1069-
{num_tests_with_crashes} test(s) with some portion expecting `CRASH`
1209+
{num_tests_with_perma_crashes} test(s) with some portion expecting permanent `CRASH`
10701210
MEDIUM PRIORITY:
1071-
{num_tests_with_failures_somewhere} test(s) with some portion `FAIL`ing, {num_subtests_with_failures_somewhere} subtests total
1072-
{num_tests_with_timeouts_somewhere} test(s) with some portion returning `TIMEOUT`/`NOTRUN`, {num_subtests_with_timeouts_somewhere} subtests total
1211+
{num_tests_with_perma_failures_somewhere} test(s) with some portion perma-`FAIL`ing, {num_subtests_with_perma_failures_somewhere} subtests total
1212+
{num_tests_with_perma_timeouts_somewhere} test(s) with some portion returning permanent `TIMEOUT`/`NOTRUN`, {num_subtests_with_perma_timeouts_somewhere} subtests total
1213+
{num_tests_with_intermittent_crashes} tests(s) with some portion expecting intermittent `CRASH`
1214+
{num_tests_with_intermittent_runner_errors} test(s) with execution reporting intermittent `ERROR`
1215+
LOW PRIORITY:
1216+
{num_tests_with_intermittent_timeouts_somewhere} test(s) with some portion intermittently returning `TIMEOUT`/`NOTRUN`, {num_subtests_with_intermittent_timeouts_somewhere} subtest(s) total
1217+
{num_tests_with_intermittent_failures_somewhere} test(s) with some portion intermittently `FAIL`ing, {num_subtests_with_intermittent_failures_somewhere} subtests total
10731218
"
10741219
);
10751220
});

0 commit comments

Comments
 (0)