diff --git a/CHANGELOG.md b/CHANGELOG.md index aa7adef4f..0cb71f636 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 block, delete it manually and re-run. (#231) ### Added +- **`turn_id` linking agent-loop API calls to a single user prompt** — a new + `compute_turn_id(model, system, messages)` helper in + `headroom/proxy/helpers.py` hashes the message prefix up to and including + the last user-text message, yielding an id that is stable across every + agent-loop iteration of one prompt but rolls over when the user sends a + new prompt (or runs `/compact`, `/clear`). `RequestLog` gained a + `turn_id: str | None` field, which is stamped at every log site + (anthropic handler bedrock + direct branches, and the streaming handler) + and surfaced as `turn_id` in `/transformations/feed`. Lets downstream + consumers (e.g. the Headroom Desktop Activity tab) aggregate savings per + user prompt rather than per API call. - **Live flush of traffic-learned patterns to CLAUDE.md / MEMORY.md** — the `TrafficLearner` now writes to agent-native context files continuously during proxy operation, not just at shutdown. A new dirty-flag debounced diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py index 974de2dd0..1c024f9a0 100644 --- a/headroom/proxy/handlers/anthropic.py +++ b/headroom/proxy/handlers/anthropic.py @@ -315,6 +315,7 @@ class AnthropicHandlerMixin: MAX_REQUEST_BODY_SIZE, _get_image_compressor, _read_request_json, + compute_turn_id, ) from headroom.proxy.models import RequestLog from headroom.proxy.modes import is_cache_mode, is_token_mode @@ -1348,6 +1349,9 @@ class AnthropicHandlerMixin: request_messages=body.get("messages") if self.config.log_full_messages else None, + turn_id=compute_turn_id( + model, body.get("system"), body.get("messages") + ), ) ) @@ -1818,6 +1822,9 @@ class AnthropicHandlerMixin: request_messages=messages if self.config.log_full_messages else None, + turn_id=compute_turn_id( + model, body.get("system"), body.get("messages") + ), ) ) diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py index f14f107dd..f6298d97e 100644 --- a/headroom/proxy/handlers/streaming.py +++ b/headroom/proxy/handlers/streaming.py @@ -13,7 +13,7 @@ import time from datetime import datetime from typing import TYPE_CHECKING, Any -from headroom.proxy.helpers import jitter_delay_ms +from headroom.proxy.helpers import compute_turn_id, jitter_delay_ms if TYPE_CHECKING: from fastapi.responses import Response, StreamingResponse @@ -1044,6 +1044,9 @@ class StreamingMixin: request_messages=body.get("messages") if self.config.log_full_messages else None, + turn_id=compute_turn_id( + model, body.get("system"), body.get("messages") + ), ) ) diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.py index c75d8436f..c50834ac7 100644 --- a/headroom/proxy/helpers.py +++ b/headroom/proxy/helpers.py @@ -8,6 +8,7 @@ Extracted from server.py for maintainability. from __future__ import annotations +import hashlib import json import logging import random @@ -278,3 +279,67 @@ async def _read_request_json(request: Request) -> dict[str, Any]: if not isinstance(result, dict): raise ValueError("Request body must be a JSON object, not " + type(result).__name__) return result + + +def compute_turn_id( + model: str, + system: Any, + messages: list[dict[str, Any]] | None, +) -> str | None: + """Group all agent-loop API calls triggered by a single user prompt. + + A turn spans the user's text prompt plus every assistant tool-use and + user tool-result message the agent appends while executing that prompt. + Hashing the prefix up to and including the last user *text* message yields + an id that is stable across the turn but rolls over when the user sends a + new prompt. + + Returns None when no user-text message is present (nothing to identify). + """ + if not messages: + return None + + last_text_user_idx: int | None = None + for i in range(len(messages) - 1, -1, -1): + msg = messages[i] + if not isinstance(msg, dict) or msg.get("role") != "user": + continue + content = msg.get("content") + if isinstance(content, str) and content: + last_text_user_idx = i + break + if isinstance(content, list): + has_text = any( + isinstance(block, dict) and block.get("type") == "text" for block in content + ) + has_tool_result = any( + isinstance(block, dict) and block.get("type") == "tool_result" for block in content + ) + # An agent-loop continuation carries tool_result blocks; only a + # fresh user turn is text-only. + if has_text and not has_tool_result: + last_text_user_idx = i + break + + if last_text_user_idx is None: + return None + + prefix = messages[: last_text_user_idx + 1] + try: + prefix_json = json.dumps(prefix, sort_keys=True, default=str) + except (TypeError, ValueError): + return None + + h = hashlib.sha256() + h.update(model.encode("utf-8", errors="replace")) + h.update(b"\0") + if isinstance(system, str): + h.update(system.encode("utf-8", errors="replace")) + elif system is not None: + try: + h.update(json.dumps(system, sort_keys=True, default=str).encode("utf-8")) + except (TypeError, ValueError): + pass + h.update(b"\0") + h.update(prefix_json.encode("utf-8", errors="replace")) + return h.hexdigest()[:16] diff --git a/headroom/proxy/models.py b/headroom/proxy/models.py index 1bbf5f3fd..43ffa26f2 100644 --- a/headroom/proxy/models.py +++ b/headroom/proxy/models.py @@ -50,6 +50,11 @@ class RequestLog: response_content: str | None = None error: str | None = None + # Groups every agent-loop API call from one user prompt into a single turn. + # See ``headroom.proxy.helpers.compute_turn_id`` for the derivation. None + # when no user-text message is present in the request. + turn_id: str | None = None + # NOTE (Unit 2 follow-up): stage timings and session_id were briefly # added here but are now emitted exclusively through # ``emit_stage_timings_log`` (structured log line) and Prometheus. diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py index d16dcd9ca..8cdcbe714 100644 --- a/headroom/proxy/server.py +++ b/headroom/proxy/server.py @@ -1720,6 +1720,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: "transforms_applied": log.get("transforms_applied", []), "request_messages": log.get("request_messages"), "response_content": log.get("response_content"), + "turn_id": log.get("turn_id"), } ) diff --git a/tests/test_proxy/test_compute_turn_id.py b/tests/test_proxy/test_compute_turn_id.py new file mode 100644 index 000000000..c45f5af3b --- /dev/null +++ b/tests/test_proxy/test_compute_turn_id.py @@ -0,0 +1,156 @@ +"""Tests for ``headroom.proxy.helpers.compute_turn_id``.""" + +from __future__ import annotations + +from headroom.proxy.helpers import compute_turn_id + +MODEL = "claude-sonnet-4-5" +SYSTEM = "You are helpful." + + +def _user(text: str) -> dict: + return {"role": "user", "content": text} + + +def _assistant_tool_use(tool_id: str, name: str) -> dict: + return { + "role": "assistant", + "content": [{"type": "tool_use", "id": tool_id, "name": name, "input": {}}], + } + + +def _user_tool_result(tool_id: str, out: str) -> dict: + return { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": tool_id, "content": out}], + } + + +def test_returns_none_when_messages_empty(): + assert compute_turn_id(MODEL, SYSTEM, []) is None + assert compute_turn_id(MODEL, SYSTEM, None) is None + + +def test_returns_none_when_no_user_text_message(): + messages = [_assistant_tool_use("t1", "bash")] + assert compute_turn_id(MODEL, SYSTEM, messages) is None + + +def test_stable_across_agent_loop_iterations(): + iteration_1 = [_user("fix the bug")] + iteration_2 = iteration_1 + [ + _assistant_tool_use("t1", "read"), + _user_tool_result("t1", "file contents"), + ] + iteration_3 = iteration_2 + [ + _assistant_tool_use("t2", "edit"), + _user_tool_result("t2", "edit ok"), + ] + + id1 = compute_turn_id(MODEL, SYSTEM, iteration_1) + id2 = compute_turn_id(MODEL, SYSTEM, iteration_2) + id3 = compute_turn_id(MODEL, SYSTEM, iteration_3) + + assert id1 is not None + assert id1 == id2 == id3 + + +def test_rolls_over_on_new_user_prompt(): + turn_1 = [_user("first prompt")] + turn_2 = turn_1 + [ + _assistant_tool_use("t1", "bash"), + _user_tool_result("t1", "ok"), + _user("second prompt"), + ] + + id1 = compute_turn_id(MODEL, SYSTEM, turn_1) + id2 = compute_turn_id(MODEL, SYSTEM, turn_2) + + assert id1 != id2 + + +def test_different_model_yields_different_id(): + messages = [_user("same prompt")] + id_a = compute_turn_id("claude-sonnet-4-5", SYSTEM, messages) + id_b = compute_turn_id("claude-opus-4-7", SYSTEM, messages) + assert id_a != id_b + + +def test_different_system_yields_different_id(): + messages = [_user("same prompt")] + id_a = compute_turn_id(MODEL, "system A", messages) + id_b = compute_turn_id(MODEL, "system B", messages) + assert id_a != id_b + + +def test_accepts_list_system_prompt(): + messages = [_user("hi")] + system_list = [{"type": "text", "text": "You are helpful."}] + assert compute_turn_id(MODEL, system_list, messages) is not None + + +def test_text_block_in_list_content_is_a_user_turn(): + messages = [{"role": "user", "content": [{"type": "text", "text": "hello"}]}] + assert compute_turn_id(MODEL, SYSTEM, messages) is not None + + +def test_tool_result_only_content_is_not_a_turn_boundary(): + # A message whose only content is a tool_result is a continuation, not a + # new turn — so the function must not latch onto it. + messages = [_user_tool_result("t1", "result only")] + assert compute_turn_id(MODEL, SYSTEM, messages) is None + + +def test_returns_16_hex_chars(): + turn_id = compute_turn_id(MODEL, SYSTEM, [_user("hi")]) + assert turn_id is not None + assert len(turn_id) == 16 + int(turn_id, 16) # raises if not hex + + +def test_skips_non_dict_and_non_user_messages(): + # A non-dict entry and an assistant message must both be skipped by the + # reverse scan before it finds the real user-text message. + messages = [ + _user("the actual prompt"), + {"role": "assistant", "content": "response"}, + "not-a-dict-message-entry", + ] + assert compute_turn_id(MODEL, SYSTEM, messages) is not None + + +def test_ignores_empty_string_user_content(): + # An empty-string user content is not a real prompt; keep scanning. + messages = [_user(""), _user("the real prompt")] + hit = compute_turn_id(MODEL, SYSTEM, messages) + assert hit is not None + # Hash should match a single-message [real prompt] prefix — i.e. the + # scan stopped at "the real prompt" and included the leading empty msg + # in the hashed prefix. Either way: not None and reproducible. + assert hit == compute_turn_id(MODEL, SYSTEM, messages) + + +def test_mixed_text_and_tool_result_is_not_a_turn_boundary(): + # A user message whose content list has BOTH text and tool_result is + # treated as an agent-loop continuation (not a fresh prompt). If + # nothing else earlier qualifies, compute_turn_id returns None. + messages = [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "ok"}, + {"type": "text", "text": "and a comment"}, + ], + } + ] + assert compute_turn_id(MODEL, SYSTEM, messages) is None + + +def test_none_system_hashes_without_system_segment(): + messages = [_user("hi")] + a = compute_turn_id(MODEL, None, messages) + b = compute_turn_id(MODEL, None, messages) + assert a is not None + assert a == b + # Different-system values must still produce a different id than None. + assert a != compute_turn_id(MODEL, "some system", messages)