mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
Fixes #1095. ## Problem SmartCrusher emits **12-hex-char** hashes inside `<<ccr:HASH N_rows_offloaded>>` (and the opaque-blob `<<ccr:HASH,KIND,SIZE>>`) markers, and the compression store serves them over `GET /v1/retrieve/{hash}`. But `CCRToolInjector.scan_for_markers()` and `parse_tool_call()` in `headroom/ccr/tool_injection.py` only recognized the **24-char** hex used by the legacy bracket markers, so the two layers were out of sync: - `scan_for_markers()` returned `[]` for SmartCrusher output (injector thought no compressed content was present). - `parse_tool_call()` returned `(None, None)` for 12-char hashes. - `POST /v1/retrieve/tool_call` and the proxy auto-continue path — both route through `parse_tool_call` (`proxy/server.py`, `ccr/response_handler.py`) — returned **400**, while `GET /v1/retrieve/{12-char-hash}` worked. ## Fix (scoped to `tool_injection.py`) - **`scan_for_markers`**: add a `<<ccr:([a-f0-9]{12,24})>>` pattern matching the row-drop summary and opaque-blob marker forms. This mirrors the substring scan already used in `transforms/smart_crusher.py::_collect_ccr_hashes_from_string`. - **`parse_tool_call`**: accept the two real CCR hash lengths (12 or 24 hex) instead of requiring exactly 24. Shorter, longer, or non-hex hashes are still rejected. Legacy 24-char bracket markers and the existing `TestHashSecurityValidation` tests are unaffected (a 6-char hash is still too short, a 30-char hash still too long). ## Verification Loaded the modified module directly and confirmed: | input | before | after | |---|---|---| | `<<ccr:e21a26620105 988_rows_offloaded>>` scan | `[]` | `['e21a26620105']` | | `<<ccr:deadbeefdead,string,2.3KB>>` scan | `[]` | `['deadbeefdead']` | | `parse_tool_call` 12-char hash | `(None, None)` | `('e21a26620105', query)` | | `parse_tool_call` 24-char hash | works | works (unchanged) | | `parse_tool_call` 6-char / 30-char / non-hex | rejected | rejected | Adds `TestSmartCrusherCcrMarkers` covering both marker forms, the 12-char parse path, and a regression guard for the 24-char path.
This commit is contained in:
parent
d437d35dbb
commit
9f7f3adfea
2 changed files with 95 additions and 9 deletions
|
|
@ -202,17 +202,24 @@ class CCRToolInjector:
|
|||
# - Generic: any [... compressed ... hash=xxx] pattern
|
||||
_marker_patterns: list[re.Pattern] = field(
|
||||
default_factory=lambda: [
|
||||
# All patterns require exactly 24 hex characters for hash validation
|
||||
# CCR uses SHA256 truncated to 24 hex chars (96 bits) for collision resistance
|
||||
# Requiring exact length prevents hash spoofing attacks with shorter hashes
|
||||
# Hash length is validated by the patterns themselves. Legacy
|
||||
# bracket markers carry a 24-hex-char hash (SHA-256[:24], 96 bits
|
||||
# for collision resistance); SmartCrusher's `<<ccr:>>` markers carry
|
||||
# a 12-hex-char hash (see transforms/smart_crusher.py and
|
||||
# cache/compression_store.py). Both real lengths are accepted.
|
||||
#
|
||||
# Standard format: [N <type> compressed to M. Retrieve more: hash=xxx]
|
||||
# Matches items, lines, matches, or any other type
|
||||
re.compile(r"\[(\d+) \w+ compressed to (\d+)\. Retrieve more: hash=([a-f0-9]{24})\]"),
|
||||
# Legacy format without "to M" or "Retrieve more:" (old TextCompressor)
|
||||
re.compile(r"\[(\d+) \w+ compressed\. hash=([a-f0-9]{24})\]"),
|
||||
# Generic fallback: any compression marker with hash (exactly 24 chars)
|
||||
# Generic fallback: any bracket compression marker with hash (exactly 24 chars)
|
||||
re.compile(r"\[.*?compressed.*?hash=([a-f0-9]{24})\]", re.IGNORECASE),
|
||||
# SmartCrusher markers: the row-drop summary
|
||||
# `<<ccr:HASH N_rows_offloaded>>` and the opaque-blob form
|
||||
# `<<ccr:HASH,KIND,SIZE>>`. HASH is 12-24 hex chars, terminated by a
|
||||
# space, comma, or the closing `>>`.
|
||||
re.compile(r"<<ccr:([a-f0-9]{12,24})\b"),
|
||||
]
|
||||
)
|
||||
|
||||
|
|
@ -497,10 +504,11 @@ def parse_tool_call(
|
|||
hash_key = input_data.get("hash")
|
||||
query = input_data.get("query")
|
||||
|
||||
# Validate hash format: must be exactly 24 hex characters
|
||||
# This prevents hash spoofing attacks with malformed hashes
|
||||
# Validate hash format. SmartCrusher emits 12-hex-char hashes while legacy
|
||||
# bracket markers / the compression_store use 24-hex-char hashes; accept
|
||||
# either real length and reject anything else as malformed.
|
||||
if hash_key is not None:
|
||||
if not isinstance(hash_key, str) or len(hash_key) != 24:
|
||||
if not isinstance(hash_key, str) or len(hash_key) not in (12, 24):
|
||||
return None, None
|
||||
# Validate hex characters only
|
||||
if not all(c in "0123456789abcdef" for c in hash_key.lower()):
|
||||
|
|
|
|||
|
|
@ -319,8 +319,9 @@ class TestParseToolCall:
|
|||
class TestHashSecurityValidation:
|
||||
"""Test hash validation security measures.
|
||||
|
||||
CCR hashes must be exactly 24 hex characters (96 bits of SHA256).
|
||||
This prevents hash spoofing attacks with shorter or malformed hashes.
|
||||
CCR hashes are 12 hex chars (SmartCrusher) or 24 hex chars (legacy
|
||||
bracket markers / compression_store). Any other length or non-hex input
|
||||
is rejected to prevent hash spoofing with malformed hashes.
|
||||
"""
|
||||
|
||||
def test_rejects_short_hash(self):
|
||||
|
|
@ -375,6 +376,83 @@ class TestHashSecurityValidation:
|
|||
assert hash_key == "ABC123DEF456ABC123DEF456"
|
||||
|
||||
|
||||
class TestSmartCrusherCcrMarkers:
|
||||
"""Regression tests for issue #1095.
|
||||
|
||||
SmartCrusher emits 12-hex-char hashes inside ``<<ccr:HASH ...>>`` markers
|
||||
(the row-drop summary and the opaque-blob form). The injector must detect
|
||||
those markers and ``parse_tool_call`` must accept the 12-char hashes —
|
||||
previously both only recognized the 24-char legacy bracket markers.
|
||||
"""
|
||||
|
||||
def test_scan_detects_row_drop_marker(self):
|
||||
"""Detects ``<<ccr:HASH N_rows_offloaded>>`` (12-char hash)."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "x",
|
||||
"content": '{"kept": 12, "ccr": "<<ccr:e21a26620105 988_rows_offloaded>>"}',
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
injector = CCRToolInjector(provider="anthropic")
|
||||
hashes = injector.scan_for_markers(messages)
|
||||
|
||||
assert hashes == ["e21a26620105"]
|
||||
assert injector.has_compressed_content
|
||||
|
||||
def test_scan_detects_opaque_blob_marker(self):
|
||||
"""Detects the ``<<ccr:HASH,KIND,SIZE>>`` opaque-blob form."""
|
||||
messages = [
|
||||
{"role": "tool", "content": "<<ccr:deadbeefdead,string,2.3KB>>"},
|
||||
]
|
||||
|
||||
hashes = CCRToolInjector().scan_for_markers(messages)
|
||||
|
||||
assert hashes == ["deadbeefdead"]
|
||||
|
||||
def test_legacy_bracket_marker_still_detected(self):
|
||||
"""The 24-char legacy bracket marker keeps working alongside the new one."""
|
||||
messages = [
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "[100 items compressed to 10. Retrieve more: hash=abc123def456abc123def456]",
|
||||
},
|
||||
]
|
||||
|
||||
hashes = CCRToolInjector().scan_for_markers(messages)
|
||||
|
||||
assert hashes == ["abc123def456abc123def456"]
|
||||
|
||||
def test_parse_tool_call_accepts_12_char_hash(self):
|
||||
"""``parse_tool_call`` accepts a 12-char SmartCrusher hash."""
|
||||
tool_call = {
|
||||
"name": CCR_TOOL_NAME,
|
||||
"input": {"hash": "e21a26620105", "query": "auth middleware"},
|
||||
}
|
||||
|
||||
hash_key, query = parse_tool_call(tool_call, "anthropic")
|
||||
|
||||
assert hash_key == "e21a26620105"
|
||||
assert query == "auth middleware"
|
||||
|
||||
def test_parse_tool_call_still_accepts_24_char_hash(self):
|
||||
"""24-char legacy hashes remain valid (regression guard)."""
|
||||
tool_call = {
|
||||
"name": CCR_TOOL_NAME,
|
||||
"input": {"hash": "abc123def456abc123def456"},
|
||||
}
|
||||
|
||||
hash_key, _ = parse_tool_call(tool_call, "anthropic")
|
||||
|
||||
assert hash_key == "abc123def456abc123def456"
|
||||
|
||||
|
||||
class TestSystemInstructions:
|
||||
"""Test system instruction generation."""
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue