mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
3e976712e7
## Description
`apply_verbosity_steering` (the Anthropic output-shaping path) scans the
`system` block list to find and update an existing steering block:
```python
if isinstance(system, list):
for block in system:
if isinstance(block, dict) and block.get("text", "").startswith(_STEERING_SENTINEL):
```
`.get("text", "")` only substitutes the default when the key is
**absent**. A malformed client block with a null text (`{"type": "text",
"text": null}`) returns `None`, so `None.startswith(...)` raises
`AttributeError`. In the output-shaping treatment arm that call runs
inside `shape_request`, which is not individually guarded, so the
exception propagates and 502s the request.
The OpenAI chat sibling in the same module already defends against this
exact case (`isinstance(part.get("text"), str)`), so the Anthropic path
is the inconsistent one.
## Fix
Guard that the block text is a string before `startswith`, mirroring the
OpenAI sibling. Well-formed bodies are unchanged: the steering block is
still replaced idempotently when a level changes, or appended when
absent. The malformed block is left untouched.
## Type of Change
- [x] Bug fix (non-breaking change that fixes an issue)
- [ ] New feature (non-breaking change that adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
- [ ] Documentation update
- [ ] Performance improvement
- [ ] Code refactoring (no functional changes)
## Changes Made
- `headroom/proxy/output_steering.py`: string-guard the system block
text before `startswith` in `apply_verbosity_steering`.
- `tests/test_output_steering.py`: regression asserting a `system` list
containing a `{"text": null}` block does not crash and still appends the
steering block.
## Testing
- [x] Unit tests pass (`pytest`)
- [x] Linting passes (`ruff check .`)
- [x] Type checking passes (`mypy headroom`)
- [x] New tests added for new functionality
- [ ] Manual testing performed
### Test Output
```text
$ python -m pytest tests/test_output_steering.py -q
9 passed
# with the fix reverted, the new test fails (AttributeError on None.startswith):
$ git stash push -- headroom/proxy/output_steering.py
$ python -m pytest "tests/test_output_steering.py::test_anthropic_steering_tolerates_non_string_system_block_text" -q
1 failed
$ uvx ruff@0.15.17 check headroom/proxy/output_steering.py tests/test_output_steering.py
All checks passed!
$ uvx mypy@1.20.2 --ignore-missing-imports headroom/proxy/output_steering.py
Success: no issues found in 1 source file
```
## Real Behavior Proof
- Environment: Windows 11, Python 3.12, project venv (`uv sync --extra
proxy`), `uvx ruff@0.15.17` / `uvx mypy@1.20.2`, pytest in the venv.
- Exact command / steps: called the real `apply_verbosity_steering` with
`system=[{"type":"text","text":None},{"type":"text","text":"Real system
prompt."}]`; also confirmed the OpenAI sibling
`apply_openai_chat_verbosity_steering` handles the same shape.
- Observed result: pre-fix the Anthropic call raised `AttributeError:
'NoneType' object has no attribute 'startswith'` while the OpenAI
sibling returned True; post-fix the Anthropic call returns True, leaves
the malformed block as-is, appends the steering block, and stays
idempotent on a repeat. Ran against the actual module.
- Not tested: a live client that sends a null system block text end to
end.
## Review Readiness
- [x] I have performed a self-review
- [x] This PR is ready for human review
## Checklist
- [x] My code follows the project's style guidelines
- [x] I have performed a self-review of my code
- [x] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] New and existing unit tests pass locally with my changes
- [ ] I have updated the CHANGELOG.md if applicable
124 lines
4.7 KiB
Python
124 lines
4.7 KiB
Python
"""Tests for output verbosity steering helpers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from headroom.proxy.output_steering import (
|
|
apply_openai_responses_verbosity_steering,
|
|
apply_verbosity_steering,
|
|
replace_or_append_steering_block,
|
|
steering_text,
|
|
)
|
|
|
|
|
|
def test_replace_or_append_steering_block_replaces_existing_block() -> None:
|
|
old = steering_text(1)
|
|
new = steering_text(3)
|
|
assert old is not None
|
|
assert new is not None
|
|
updated, changed = replace_or_append_steering_block(f"System.\n\n{old}\n\nTail.", new)
|
|
|
|
assert changed is True
|
|
assert old not in updated
|
|
assert updated == f"System.\n\n{new}\n\nTail."
|
|
|
|
|
|
def test_anthropic_steering_preserves_cached_prefix_block() -> None:
|
|
cached = {
|
|
"type": "text",
|
|
"text": "Big system prompt.",
|
|
"cache_control": {"type": "ephemeral"},
|
|
}
|
|
body = {"system": [cached.copy()]}
|
|
|
|
assert apply_verbosity_steering(body, 2) is True
|
|
assert body["system"][0] == cached
|
|
assert body["system"][1] == {"type": "text", "text": steering_text(2)}
|
|
|
|
|
|
def test_anthropic_steering_tolerates_non_string_system_block_text() -> None:
|
|
# A malformed client block ({"type": "text", "text": null}) must not crash
|
|
# `.startswith` and 500 the request; steering is still appended. The OpenAI
|
|
# chat sibling already guards this exact case.
|
|
body = {
|
|
"system": [
|
|
{"type": "text", "text": None},
|
|
{"type": "text", "text": "Real system prompt."},
|
|
]
|
|
}
|
|
|
|
assert apply_verbosity_steering(body, 2) is True
|
|
# The malformed block is left as-is and a steering block is appended.
|
|
assert body["system"][0] == {"type": "text", "text": None}
|
|
assert body["system"][-1] == {"type": "text", "text": steering_text(2)}
|
|
|
|
|
|
def test_openai_responses_steering_is_idempotent() -> None:
|
|
body = {"instructions": "System."}
|
|
|
|
assert apply_openai_responses_verbosity_steering(body, 2) is True
|
|
snapshot = body.copy()
|
|
assert apply_openai_responses_verbosity_steering(body, 2) is False
|
|
assert body == snapshot
|
|
|
|
|
|
def test_openai_chat_steering_appends_to_system_message() -> None:
|
|
from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering
|
|
|
|
body = {
|
|
"messages": [
|
|
{"role": "system", "content": "You are helpful."},
|
|
{"role": "user", "content": "hi"},
|
|
]
|
|
}
|
|
assert apply_openai_chat_verbosity_steering(body, 2) is True
|
|
sys_content = body["messages"][0]["content"]
|
|
assert "You are helpful." in sys_content
|
|
assert steering_text(2) in sys_content
|
|
# Other messages and ordering are untouched.
|
|
assert body["messages"][1] == {"role": "user", "content": "hi"}
|
|
assert [m["role"] for m in body["messages"]] == ["system", "user"]
|
|
|
|
|
|
def test_openai_chat_steering_is_idempotent_and_swaps_level() -> None:
|
|
from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering
|
|
|
|
body = {"messages": [{"role": "system", "content": "S."}]}
|
|
assert apply_openai_chat_verbosity_steering(body, 2) is True
|
|
first = body["messages"][0]["content"]
|
|
# Same level again: no change.
|
|
assert apply_openai_chat_verbosity_steering(body, 2) is False
|
|
assert body["messages"][0]["content"] == first
|
|
# Different level: replace, still exactly one block.
|
|
assert apply_openai_chat_verbosity_steering(body, 4) is True
|
|
swapped = body["messages"][0]["content"]
|
|
assert steering_text(4) in swapped
|
|
assert swapped.count("<headroom_output_shaping>") == 1
|
|
|
|
|
|
def test_openai_chat_steering_inserts_system_when_absent() -> None:
|
|
from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering
|
|
|
|
body = {"messages": [{"role": "user", "content": "hi"}]}
|
|
assert apply_openai_chat_verbosity_steering(body, 3) is True
|
|
assert body["messages"][0]["role"] == "system"
|
|
assert body["messages"][0]["content"] == steering_text(3)
|
|
assert body["messages"][1] == {"role": "user", "content": "hi"}
|
|
|
|
|
|
def test_openai_chat_steering_handles_list_content() -> None:
|
|
from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering
|
|
|
|
body = {"messages": [{"role": "system", "content": [{"type": "text", "text": "base"}]}]}
|
|
assert apply_openai_chat_verbosity_steering(body, 1) is True
|
|
parts = body["messages"][0]["content"]
|
|
assert parts[0] == {"type": "text", "text": "base"}
|
|
assert parts[1]["type"] == "text"
|
|
assert parts[1]["text"] == steering_text(1)
|
|
|
|
|
|
def test_openai_chat_steering_level_zero_is_noop() -> None:
|
|
from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering
|
|
|
|
body = {"messages": [{"role": "system", "content": "S."}]}
|
|
assert apply_openai_chat_verbosity_steering(body, 0) is False
|
|
assert body["messages"][0]["content"] == "S."
|