mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
feat(evals): add zero-cost tool schema compaction integrity eval (#817)
## Summary - Adds `evaluate_tool_schema_compaction()` and `generate_tool_schema_cases()` to `CompressionOnlyRunner` - Four built-in cases cover the property-name vs annotation-key distinction: `title`, `deprecated`, `readOnly`, and all four at once - Each case asserts: byte count shrinks (annotations stripped), all `must_preserve` property names survive in `properties`, no `required` entry points to a stripped key, root-level schema annotations (`$schema`, `title`) are dropped - Wires the new eval into `.github/workflows/eval.yml` alongside the existing CCR round-trip smoke step — runs on every PR touching `headroom/transforms/**`, `headroom/evals/**`, or `headroom/compress.py`, at zero API cost ## Motivation PR #785 fixed a bug where the compaction pass stripped property *names* that happened to match DROP_KEYS (e.g. a field literally called `title`). This eval encodes the invariant that fix established so future changes to the compaction logic can't silently regress it. ## Test plan - [ ] `pytest tests/test_evals_metrics.py::test_tool_schema_compaction_integrity` — all 4 cases pass, `total_tokens_saved > 0` - [ ] CI smoke step "Run tool schema compaction integrity eval (zero cost)" passes with no API key required ## Real behavior proof ``` $ pytest tests/test_evals_metrics.py::test_tool_schema_compaction_integrity -v PASSED [100%] 1 passed in 0.53s ``` Zero API calls, zero cost. Runs in under 1 second. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Ashish
GitHub
parent
93c69372e6
commit
53a08c63bf
3 changed files with 305 additions and 0 deletions
11
.github/workflows/eval.yml
vendored
11
.github/workflows/eval.yml
vendored
|
|
@ -54,6 +54,17 @@ jobs:
|
|||
print(f'CCR Round-trip: {result.passed_cases}/{result.total_cases} passed')
|
||||
assert result.passed, f'CCR failures: {result.errors}'
|
||||
"
|
||||
|
||||
- name: Run tool schema compaction integrity eval (zero cost)
|
||||
run: |
|
||||
python -c "
|
||||
from headroom.evals.runners.compression_only import CompressionOnlyRunner
|
||||
runner = CompressionOnlyRunner()
|
||||
result = runner.evaluate_tool_schema_compaction()
|
||||
print(f'Tool schema compaction: {result.passed_cases}/{result.total_cases} passed, {result.total_tokens_saved} annotation tokens stripped')
|
||||
assert result.passed, f'Schema compaction failures: {result.errors}'
|
||||
"
|
||||
|
||||
# OPENAI_API_KEY is intentionally not set in the public OSS repo
|
||||
# (the secret list is empty). The CCR round-trip step above is the
|
||||
# mandatory gate; this step only runs when an operator has wired
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Used for:
|
|||
- CCR lossless round-trip verification
|
||||
- Information retention (probe facts survive compression)
|
||||
- Needle retention (specific values preserved in compressed output)
|
||||
- Tool schema compaction integrity (property names survive annotation stripping)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -335,3 +336,275 @@ class CompressionOnlyRunner:
|
|||
)
|
||||
|
||||
return cases[:n]
|
||||
|
||||
def generate_tool_schema_cases(self) -> list[dict[str, Any]]:
|
||||
"""Generate tool schema test cases for compaction integrity verification.
|
||||
|
||||
Each case exercises a different way a DROP_KEY can appear as a
|
||||
property *name* inside a JSON Schema `properties` object.
|
||||
The cases also include annotation keys at the schema level so we
|
||||
can verify those ARE still stripped.
|
||||
"""
|
||||
return [
|
||||
{
|
||||
"id": "schema_title_property",
|
||||
"description": "property named 'title' must survive; schema-level title must be dropped",
|
||||
"payload": {
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "eval_cells",
|
||||
"description": "Evaluate notebook cells.",
|
||||
"parameters": {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "EvalCellsParameters",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"cells": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"title": "CellItem",
|
||||
"properties": {
|
||||
"language": {"type": "string"},
|
||||
"code": {"type": "string"},
|
||||
"title": {"type": "string"},
|
||||
},
|
||||
"required": ["language", "code", "title"],
|
||||
},
|
||||
}
|
||||
},
|
||||
"required": ["cells"],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"must_preserve": ["title"],
|
||||
"must_drop_schema_annotations": True,
|
||||
},
|
||||
{
|
||||
"id": "schema_deprecated_property",
|
||||
"description": "property named 'deprecated' must survive",
|
||||
"payload": {
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "list_apis",
|
||||
"description": "List available APIs with their status.",
|
||||
"parameters": {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "ListApisParameters",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"deprecated": {
|
||||
"type": "boolean",
|
||||
"description": "Include deprecated APIs in results.",
|
||||
},
|
||||
"name": {"type": "string"},
|
||||
},
|
||||
"required": ["deprecated"],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"must_preserve": ["deprecated"],
|
||||
"must_drop_schema_annotations": True,
|
||||
},
|
||||
{
|
||||
"id": "schema_readonly_property",
|
||||
"description": "property named 'readOnly' must survive",
|
||||
"payload": {
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "update_field",
|
||||
"description": "Update a field in a record.",
|
||||
"parameters": {
|
||||
"title": "UpdateFieldParameters",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"field_name": {"type": "string"},
|
||||
"value": {"type": "string"},
|
||||
"readOnly": {
|
||||
"type": "boolean",
|
||||
"description": "Whether the field is read-only.",
|
||||
},
|
||||
},
|
||||
"required": ["field_name", "value", "readOnly"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"must_preserve": ["readOnly"],
|
||||
"must_drop_schema_annotations": True,
|
||||
},
|
||||
{
|
||||
"id": "schema_multiple_collisions",
|
||||
"description": "multiple DROP_KEY collisions in one schema",
|
||||
"payload": {
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "create_field",
|
||||
"description": "Create a schema field descriptor.",
|
||||
"parameters": {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "CreateFieldParameters",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"deprecated": {"type": "boolean"},
|
||||
"examples": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
},
|
||||
"readOnly": {"type": "boolean"},
|
||||
},
|
||||
"required": ["title", "deprecated", "examples", "readOnly"],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"must_preserve": ["title", "deprecated", "examples", "readOnly"],
|
||||
"must_drop_schema_annotations": True,
|
||||
},
|
||||
]
|
||||
|
||||
def evaluate_tool_schema_compaction(
|
||||
self,
|
||||
cases: list[dict[str, Any]] | None = None,
|
||||
) -> CompressionOnlyResult:
|
||||
"""Verify tool schema compaction preserves property names that collide with DROP_KEYS.
|
||||
|
||||
The compaction pass must never strip a key that appears as a property
|
||||
*name* under a JSON Schema `properties` object, even if the same key
|
||||
is in the annotation drop-list (title, deprecated, readOnly, examples, …).
|
||||
|
||||
Assertions per case:
|
||||
- token count is smaller after compaction (annotations were stripped)
|
||||
- every property name listed in `must_preserve` is present in the
|
||||
compacted schema's `properties` dict
|
||||
- every `required` array is a subset of the surviving `properties` keys
|
||||
(no dangling required entry pointing at a stripped property)
|
||||
- schema-level annotations ($schema, title at root level) ARE dropped
|
||||
"""
|
||||
from headroom.proxy.handlers.openai import _compact_openai_responses_tools
|
||||
|
||||
if cases is None:
|
||||
cases = self.generate_tool_schema_cases()
|
||||
|
||||
start_time = time.time()
|
||||
passed = 0
|
||||
failed = 0
|
||||
total_original = 0
|
||||
total_compressed = 0
|
||||
details: list[dict[str, Any]] = []
|
||||
errors: list[str] = []
|
||||
|
||||
for case in cases:
|
||||
case_id = case["id"]
|
||||
payload = case["payload"]
|
||||
must_preserve: list[str] = case.get("must_preserve", [])
|
||||
must_drop_schema_annotations: bool = case.get("must_drop_schema_annotations", False)
|
||||
|
||||
original_bytes = len(json.dumps(payload).encode())
|
||||
total_original += original_bytes
|
||||
|
||||
try:
|
||||
compacted, modified, before_bytes, after_bytes = _compact_openai_responses_tools(
|
||||
payload
|
||||
)
|
||||
total_compressed += after_bytes if modified else original_bytes
|
||||
|
||||
case_errors: list[str] = []
|
||||
|
||||
if not modified:
|
||||
case_errors.append(
|
||||
"compaction reported no modification (annotations not stripped)"
|
||||
)
|
||||
|
||||
for tool in compacted.get("tools", []):
|
||||
params = tool.get("parameters", {})
|
||||
_check_properties_recursive(params, must_preserve, tool["name"], case_errors)
|
||||
|
||||
if must_drop_schema_annotations:
|
||||
for tool in compacted.get("tools", []):
|
||||
params = tool.get("parameters", {})
|
||||
for ann_key in ("$schema", "title"):
|
||||
if ann_key in params:
|
||||
case_errors.append(
|
||||
f"tool '{tool['name']}': schema annotation '{ann_key}' "
|
||||
f"was not stripped from parameters root"
|
||||
)
|
||||
|
||||
is_pass = len(case_errors) == 0
|
||||
if is_pass:
|
||||
passed += 1
|
||||
else:
|
||||
failed += 1
|
||||
errors.extend(f"[{case_id}] {e}" for e in case_errors)
|
||||
|
||||
details.append(
|
||||
{
|
||||
"id": case_id,
|
||||
"passed": is_pass,
|
||||
"original_bytes": before_bytes,
|
||||
"compacted_bytes": after_bytes,
|
||||
"compression_ratio": 1 - (after_bytes / before_bytes)
|
||||
if before_bytes > 0
|
||||
else 0,
|
||||
"errors": case_errors,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
failed += 1
|
||||
total_compressed += original_bytes
|
||||
errors.append(f"[{case_id}] unexpected exception: {exc}")
|
||||
details.append({"id": case_id, "passed": False, "error": str(exc)})
|
||||
|
||||
total_cases = passed + failed
|
||||
ratios = [d.get("compression_ratio", 0) for d in details if "compression_ratio" in d]
|
||||
|
||||
return CompressionOnlyResult(
|
||||
benchmark="tool_schema_compaction",
|
||||
total_cases=total_cases,
|
||||
passed_cases=passed,
|
||||
failed_cases=failed,
|
||||
accuracy_rate=passed / total_cases if total_cases > 0 else 0.0,
|
||||
avg_compression_ratio=sum(ratios) / len(ratios) if ratios else 0.0,
|
||||
total_original_tokens=total_original // 4,
|
||||
total_compressed_tokens=total_compressed // 4,
|
||||
total_tokens_saved=(total_original - total_compressed) // 4,
|
||||
duration_seconds=time.time() - start_time,
|
||||
details=details,
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
|
||||
def _check_properties_recursive(
|
||||
schema: Any,
|
||||
must_preserve: list[str],
|
||||
tool_name: str,
|
||||
errors: list[str],
|
||||
) -> None:
|
||||
"""Walk a JSON Schema object and assert that must_preserve keys survive inside `properties`."""
|
||||
if not isinstance(schema, dict):
|
||||
return
|
||||
|
||||
properties = schema.get("properties")
|
||||
if isinstance(properties, dict):
|
||||
required = schema.get("required", [])
|
||||
for key in must_preserve:
|
||||
if key in required and key not in properties:
|
||||
errors.append(
|
||||
f"tool '{tool_name}': property '{key}' is in `required` but was "
|
||||
f"stripped from `properties` by compaction"
|
||||
)
|
||||
for sub_schema in properties.values():
|
||||
_check_properties_recursive(sub_schema, must_preserve, tool_name, errors)
|
||||
|
||||
items = schema.get("items")
|
||||
if isinstance(items, dict):
|
||||
_check_properties_recursive(items, must_preserve, tool_name, errors)
|
||||
|
|
|
|||
|
|
@ -130,3 +130,24 @@ def test_information_recall_reports_preserved_and_missing_facts() -> None:
|
|||
empty_original = metrics.compute_information_recall("No facts here", "Still none", ["Alice"])
|
||||
assert empty_original["facts_in_original"] == 0
|
||||
assert empty_original["recall"] == 1.0
|
||||
|
||||
|
||||
def test_tool_schema_compaction_integrity() -> None:
|
||||
"""Property names that collide with DROP_KEYS must survive schema compaction.
|
||||
|
||||
Runs the full CompressionOnlyRunner.evaluate_tool_schema_compaction() path
|
||||
against the built-in cases and asserts zero failures. This is zero-cost
|
||||
(no API calls) and safe for CI smoke runs.
|
||||
"""
|
||||
from headroom.evals.runners.compression_only import CompressionOnlyRunner
|
||||
|
||||
runner = CompressionOnlyRunner()
|
||||
result = runner.evaluate_tool_schema_compaction()
|
||||
|
||||
assert result.passed, (
|
||||
f"Tool schema compaction integrity failures "
|
||||
f"({result.failed_cases}/{result.total_cases}):\n" + "\n".join(result.errors)
|
||||
)
|
||||
assert result.total_cases == 4, f"Expected 4 built-in cases, got {result.total_cases}"
|
||||
# Annotations were stripped so byte count must have shrunk.
|
||||
assert result.total_tokens_saved > 0, "Expected at least some annotation tokens to be stripped"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue