diff --git a/headroom/evals/memory/judge.py b/headroom/evals/memory/judge.py index be468e844..4e711d352 100644 --- a/headroom/evals/memory/judge.py +++ b/headroom/evals/memory/judge.py @@ -201,7 +201,8 @@ def _parse_judge_response(text: str) -> tuple[float, str]: Tuple of (score, reasoning). """ reasoning = "" - score = 3.0 # Default to middle score if parsing fails + score: float | None = None + parsed = False lines = text.strip().split("\n") @@ -222,9 +223,20 @@ def _parse_judge_response(text: str) -> tuple[float, str]: score = float(match.group(1)) # Clamp to valid range score = max(1.0, min(5.0, score)) + parsed = True except ValueError: logger.warning(f"Could not parse score from: {score_text}") + if not parsed: + # Default to a failing score so unparseable judge output doesn't + # silently pass downstream `judge_score >= 3.0` checks. + logger.warning( + f"Could not parse a score from judge response, defaulting to 0.0 (fail): {text!r}" + ) + score = 0.0 + + assert score is not None + # If no explicit reasoning found, use the whole text if not reasoning: reasoning = text.strip() diff --git a/tests/test_memory_eval.py b/tests/test_memory_eval.py index 0f57021da..d422cf1b0 100644 --- a/tests/test_memory_eval.py +++ b/tests/test_memory_eval.py @@ -180,6 +180,19 @@ Score: 3.5""" score, _ = _parse_judge_response(response) assert score == 1.0 + def test_parse_judge_response_unparseable_defaults_to_failing_score(self): + """Unparseable judge output must default below the pass threshold. + + Regression test for #1890: a missing/garbled "Score:" line used to + default to 3.0, which is exactly the `judge_score >= 3.0` pass + threshold in before_after.py, silently marking unparseable judge + responses as passing. + """ + response = "The model's response looks reasonable overall." + + score, _ = _parse_judge_response(response) + assert score < 3.0 + def test_simple_judge_exact_match(self): """Test simple judge with exact match.""" score, reasoning = simple_judge(