From 19531b69d1f2fa01bbda55fc8fc3fceea6574c87 Mon Sep 17 00:00:00 2001 From: spapi Date: Fri, 3 Jul 2026 10:55:35 +0200 Subject: [PATCH 1/3] Fix spacing issues with delays calculation and character units --- simulstream/metrics/scorers/latency/mwersegmenter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simulstream/metrics/scorers/latency/mwersegmenter.py b/simulstream/metrics/scorers/latency/mwersegmenter.py index 3a13b8d..75fb40f 100644 --- a/simulstream/metrics/scorers/latency/mwersegmenter.py +++ b/simulstream/metrics/scorers/latency/mwersegmenter.py @@ -123,7 +123,7 @@ def _tokenize(self, text: List[str]) -> List[str]: encoded = [" ".join(self.segmenter.encode(p)) for p in pieces] tokenized_text.append(" ### ".join(encoded)) else: - tokenized_text.append(" ".join(self.segmenter.encode(text[i].strip()))) + tokenized_text.append(" ".join(self.segmenter.encode(text[i]))) return "\n".join(tokenized_text) else: return "\n".join(text) From 9358b21f89995a621e5af2860ec809991bcffe02 Mon Sep 17 00:00:00 2001 From: spapi Date: Fri, 3 Jul 2026 11:10:08 +0200 Subject: [PATCH 2/3] Strip whitespaces alignment --- .../metrics/scorers/latency/mwersegmenter.py | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/simulstream/metrics/scorers/latency/mwersegmenter.py b/simulstream/metrics/scorers/latency/mwersegmenter.py index 75fb40f..3f8a4ba 100644 --- a/simulstream/metrics/scorers/latency/mwersegmenter.py +++ b/simulstream/metrics/scorers/latency/mwersegmenter.py @@ -123,7 +123,7 @@ def _tokenize(self, text: List[str]) -> List[str]: encoded = [" ".join(self.segmenter.encode(p)) for p in pieces] tokenized_text.append(" ### ".join(encoded)) else: - tokenized_text.append(" ".join(self.segmenter.encode(text[i]))) + tokenized_text.append(" ".join(self.segmenter.encode(text[i].strip()))) return "\n".join(tokenized_text) else: return "\n".join(text) @@ -133,6 +133,24 @@ def score(self, samples: List[LatencyScoringSample]) -> LatencyScores: for sample in samples: assert sample.reference is not None, "Cannot realign hypothesis to missing reference" + ideal_delays = sample.hypothesis.ideal_delays + computational_aware_delays = sample.hypothesis.computational_aware_delays + if self.segmenter is not None: + # _tokenize strips final_text; trim delays to match the stripped version + final_text = sample.hypothesis.final_text + leading = len(final_text) - len(final_text.lstrip()) + trailing = len(final_text) - len(final_text.rstrip()) + end = len(ideal_delays) - trailing if trailing > 0 else None + ideal_delays = ideal_delays[leading:end] + computational_aware_delays = computational_aware_delays[leading:end] + + if not ideal_delays: + resegmented_samples.append(ResegmentedLatencyScoringSample( + sample.audio_name, + [OutputWithDelays("", [], []) for _ in sample.reference], + sample.reference)) + continue + hypo = self._tokenize([sample.hypothesis.final_text]) refs = self._tokenize( [sentence_def.content for sentence_def in sample.reference]) @@ -148,10 +166,10 @@ def score(self, samples: List[LatencyScoringSample]) -> LatencyScores: hypo.replace(" ", "").replace("_", " ") for hypo in resegmented_hypos] ideal_delays_splits = self._split_delays_by_segmented_text( - sample.hypothesis.ideal_delays, + ideal_delays, resegmented_hypos) computational_aware_delays_splits = self._split_delays_by_segmented_text( - sample.hypothesis.computational_aware_delays, + computational_aware_delays, resegmented_hypos) assert len(ideal_delays_splits) == len(computational_aware_delays_splits) From acc89eea49752276ab93b285f0ac8eb208c0aa93 Mon Sep 17 00:00:00 2001 From: spapi Date: Fri, 3 Jul 2026 17:41:28 +0200 Subject: [PATCH 3/3] Revert "Strip whitespaces alignment" This reverts commit 9358b21f89995a621e5af2860ec809991bcffe02. --- .../metrics/scorers/latency/mwersegmenter.py | 24 +++---------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/simulstream/metrics/scorers/latency/mwersegmenter.py b/simulstream/metrics/scorers/latency/mwersegmenter.py index 3f8a4ba..75fb40f 100644 --- a/simulstream/metrics/scorers/latency/mwersegmenter.py +++ b/simulstream/metrics/scorers/latency/mwersegmenter.py @@ -123,7 +123,7 @@ def _tokenize(self, text: List[str]) -> List[str]: encoded = [" ".join(self.segmenter.encode(p)) for p in pieces] tokenized_text.append(" ### ".join(encoded)) else: - tokenized_text.append(" ".join(self.segmenter.encode(text[i].strip()))) + tokenized_text.append(" ".join(self.segmenter.encode(text[i]))) return "\n".join(tokenized_text) else: return "\n".join(text) @@ -133,24 +133,6 @@ def score(self, samples: List[LatencyScoringSample]) -> LatencyScores: for sample in samples: assert sample.reference is not None, "Cannot realign hypothesis to missing reference" - ideal_delays = sample.hypothesis.ideal_delays - computational_aware_delays = sample.hypothesis.computational_aware_delays - if self.segmenter is not None: - # _tokenize strips final_text; trim delays to match the stripped version - final_text = sample.hypothesis.final_text - leading = len(final_text) - len(final_text.lstrip()) - trailing = len(final_text) - len(final_text.rstrip()) - end = len(ideal_delays) - trailing if trailing > 0 else None - ideal_delays = ideal_delays[leading:end] - computational_aware_delays = computational_aware_delays[leading:end] - - if not ideal_delays: - resegmented_samples.append(ResegmentedLatencyScoringSample( - sample.audio_name, - [OutputWithDelays("", [], []) for _ in sample.reference], - sample.reference)) - continue - hypo = self._tokenize([sample.hypothesis.final_text]) refs = self._tokenize( [sentence_def.content for sentence_def in sample.reference]) @@ -166,10 +148,10 @@ def score(self, samples: List[LatencyScoringSample]) -> LatencyScores: hypo.replace(" ", "").replace("_", " ") for hypo in resegmented_hypos] ideal_delays_splits = self._split_delays_by_segmented_text( - ideal_delays, + sample.hypothesis.ideal_delays, resegmented_hypos) computational_aware_delays_splits = self._split_delays_by_segmented_text( - computational_aware_delays, + sample.hypothesis.computational_aware_delays, resegmented_hypos) assert len(ideal_delays_splits) == len(computational_aware_delays_splits)