mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
test: align test_error_detection with Phase 3e.1 bug fixes
This test pinned the pre-3e.1 behavior on three lines that the new KeywordDetector intentionally changes: 1. `'token'` was asserted to be in SECURITY_KEYWORDS. It was dropped from the security set in 3e.1 because it false-positived on every LLM-token reference in our own product. Updated to assert the new set (`security|password|auth|secret`) and explicitly that `token` is gone. 2. SECURITY_PATTERN was tested via "rotate the auth token" (matched via the now-removed `token` keyword). Now tested via "rotate the auth header" (matches via `auth`, which is the real security signal) plus a negative assertion that LLM-metric strings no longer fire. 3. ERROR_PATTERN test added an assertion that "Connection timeout" now flags as an error (3e.1 fixed the keyword/regex drift). PRIORITY_PATTERNS_TEXT indices were also corrected: the Rust-supplied markdown_prefixes table is ordered `# `, `## `, `### `, `#### `, `**`, `> ` (six prefixes) so the bold/blockquote assertions live at indices 6 and 7, not 3 and 4. New `# ` and `## ` checks pin the lower indices. Each diverging assertion carries a `fixed_in_3e1` comment so the audit trail stays clear.
This commit is contained in:
parent
12c2665531
commit
c39d9fe13b
1 changed files with 21 additions and 5 deletions
|
|
@ -181,22 +181,38 @@ def test_detect_content_type_respects_priority_order() -> None:
|
|||
|
||||
def test_error_detection_keywords_patterns_and_indicator_helper() -> None:
|
||||
assert {"error", "failed", "critical"} <= ERROR_KEYWORDS
|
||||
# fixed_in_3e1: ERROR_KEYWORDS canonically had timeout/abort/denied/rejected
|
||||
# but the regex omitted them; the Rust port + Python shim now align.
|
||||
assert {"timeout", "abort", "denied", "rejected"} <= ERROR_KEYWORDS
|
||||
assert {"warning", "todo", "fix"} <= IMPORTANCE_KEYWORDS
|
||||
assert {"security", "password", "token"} <= SECURITY_KEYWORDS
|
||||
# fixed_in_3e1: 'token' was dropped from SECURITY_KEYWORDS because it
|
||||
# false-positived on every LLM-token reference (input_tokens, etc.) in
|
||||
# an LLM-proxy product. 'auth' carries the real security signal.
|
||||
assert {"security", "password", "auth", "secret"} <= SECURITY_KEYWORDS
|
||||
assert "token" not in SECURITY_KEYWORDS
|
||||
assert ERROR_INDICATOR_KEYWORDS[0] == "error"
|
||||
|
||||
assert ERROR_PATTERN.search("Fatal error occurred")
|
||||
# fixed_in_3e1: timeout now matched by ERROR_PATTERN regex.
|
||||
assert ERROR_PATTERN.search("Connection timeout occurred")
|
||||
assert WARNING_PATTERN.search("warning: be careful")
|
||||
assert IMPORTANCE_PATTERN.search("TODO fix this hack")
|
||||
assert SECURITY_PATTERN.search("rotate the auth token")
|
||||
# fixed_in_3e1: pre-3e1 this matched via 'token'; now matches via 'auth'.
|
||||
assert SECURITY_PATTERN.search("rotate the auth header")
|
||||
# fixed_in_3e1: lone 'token' references no longer trigger security routing.
|
||||
assert SECURITY_PATTERN.search("input_tokens=512 output_tokens=128") is None
|
||||
|
||||
assert PRIORITY_PATTERNS_SEARCH[:3] == [ERROR_PATTERN, WARNING_PATTERN, IMPORTANCE_PATTERN]
|
||||
assert PRIORITY_PATTERNS_DIFF == [ERROR_PATTERN, IMPORTANCE_PATTERN, SECURITY_PATTERN]
|
||||
assert PRIORITY_PATTERNS_TEXT[0] is ERROR_PATTERN
|
||||
assert PRIORITY_PATTERNS_TEXT[1] is IMPORTANCE_PATTERN
|
||||
assert PRIORITY_PATTERNS_TEXT[2].match("## Header")
|
||||
assert PRIORITY_PATTERNS_TEXT[3].match("**Bold")
|
||||
assert PRIORITY_PATTERNS_TEXT[4].match("> quote")
|
||||
# The Rust-supplied markdown_prefixes order is `# `, `## `, `### `, `#### `,
|
||||
# `**`, `> ` (see KeywordRegistry::default_set). Index 2 is `# ` not `## `,
|
||||
# so anchor each assertion on the prefix it actually owns.
|
||||
assert PRIORITY_PATTERNS_TEXT[2].match("# Top-level heading")
|
||||
assert PRIORITY_PATTERNS_TEXT[3].match("## Subheading")
|
||||
assert PRIORITY_PATTERNS_TEXT[6].match("**Bold")
|
||||
assert PRIORITY_PATTERNS_TEXT[7].match("> quote")
|
||||
|
||||
assert content_has_error_indicators("TRACEBACK: Fatal crash in worker") is True
|
||||
assert content_has_error_indicators("Everything completed successfully") is False
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue