mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
4b1c449c73
## Summary - support CRLF (`\r\n\r\n`) SSE event terminators in the byte-buffer parser - parse completed SSE events with `splitlines()` so LF and CRLF line endings are handled consistently - add a regression test for CRLF-terminated SSE events ## Why SSE streams may be emitted with CRLF line endings by HTTP stacks. The existing byte-buffer parser only looked for `\n\n`, so a complete CRLF-terminated event could remain buffered and never reach usage/event parsing. ## Validation - `.venv/bin/pytest tests/test_sse_utf8_split.py tests/test_streaming_usage_parser.py -q` - `.venv/bin/ruff check headroom/proxy/helpers.py tests/test_sse_utf8_split.py` ## Risk Low. The change is isolated to complete-event boundary detection and keeps the existing invalid UTF-8 behavior loud for complete events.
107 lines
3.9 KiB
Python
107 lines
3.9 KiB
Python
"""PR-A8 / P1-8: bytes-level SSE buffer survives multi-byte splits.
|
|
|
|
Pre-A8 the proxy decoded each chunk via ``chunk.decode("utf-8",
|
|
errors="ignore")``. A 4-byte emoji or 3-byte CJK character split across
|
|
two TCP reads was silently dropped, corrupting the downstream parser's
|
|
view of the stream. The fix moves event-boundary detection into bytes
|
|
and decodes only complete events.
|
|
|
|
These tests pin:
|
|
- ``parse_sse_events_from_byte_buffer`` preserves emoji split mid-bytes.
|
|
- Same for CJK chars.
|
|
- A complete event with invalid UTF-8 raises (operator-visible diagnostic).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from headroom.proxy.helpers import parse_sse_events_from_byte_buffer
|
|
|
|
|
|
def _emit_event(name: str, data: str) -> bytes:
|
|
return f"event: {name}\ndata: {data}\n\n".encode()
|
|
|
|
|
|
def test_emoji_split_across_chunks_preserved() -> None:
|
|
"""Fire emoji `🔥` split mid-byte: 4 bytes total, split 2/2."""
|
|
# `ensure_ascii=False` keeps the emoji as raw UTF-8 bytes on the
|
|
# wire — that's the case where chunk-boundary splits actually
|
|
# corrupt content with the old `errors="ignore"` decoder.
|
|
payload = json.dumps({"type": "text_delta", "text": "fire 🔥 here"}, ensure_ascii=False)
|
|
full = _emit_event("content_block_delta", payload)
|
|
# Split somewhere inside the 4-byte emoji. The emoji `🔥` is
|
|
# `\xf0\x9f\x94\xa5` — find that sequence and split mid-bytes.
|
|
emoji_bytes = "🔥".encode()
|
|
assert emoji_bytes == b"\xf0\x9f\x94\xa5"
|
|
idx = full.find(emoji_bytes)
|
|
assert idx > 0
|
|
# Split halfway through the emoji.
|
|
chunk_a = full[: idx + 2]
|
|
chunk_b = full[idx + 2 :]
|
|
|
|
buf = bytearray()
|
|
buf.extend(chunk_a)
|
|
# First chunk should not produce any complete events because the
|
|
# event terminator `\n\n` is in chunk_b — but more importantly,
|
|
# the helper must not corrupt the partial emoji bytes.
|
|
events = parse_sse_events_from_byte_buffer(buf)
|
|
assert events == []
|
|
# Buffer still contains the partial emoji + everything before it.
|
|
assert b"\xf0\x9f" in bytes(buf)
|
|
|
|
buf.extend(chunk_b)
|
|
events = parse_sse_events_from_byte_buffer(buf)
|
|
assert len(events) == 1
|
|
name, data_str = events[0]
|
|
assert name == "content_block_delta"
|
|
parsed = json.loads(data_str)
|
|
assert parsed["text"] == "fire 🔥 here"
|
|
|
|
|
|
def test_cjk_split_across_chunks_preserved() -> None:
|
|
"""CJK `語` is 3 bytes (e8 aa 9e); split 1/2 across two reads."""
|
|
payload = json.dumps({"type": "text_delta", "text": "日本語テスト"}, ensure_ascii=False)
|
|
full = _emit_event("content_block_delta", payload)
|
|
needle = "語".encode()
|
|
assert needle == b"\xe8\xaa\x9e"
|
|
idx = full.find(needle)
|
|
assert idx > 0
|
|
chunk_a = full[: idx + 1]
|
|
chunk_b = full[idx + 1 :]
|
|
|
|
buf = bytearray()
|
|
buf.extend(chunk_a)
|
|
events = parse_sse_events_from_byte_buffer(buf)
|
|
assert events == [] # no complete event yet
|
|
|
|
buf.extend(chunk_b)
|
|
events = parse_sse_events_from_byte_buffer(buf)
|
|
assert len(events) == 1
|
|
_, data_str = events[0]
|
|
parsed = json.loads(data_str)
|
|
assert parsed["text"] == "日本語テスト"
|
|
|
|
|
|
def test_crlf_terminated_event_is_parsed() -> None:
|
|
"""SSE permits CRLF event terminators as well as LF terminators."""
|
|
buf = bytearray(b'event: message\r\ndata: {"ok": true}\r\n\r\n')
|
|
|
|
events = parse_sse_events_from_byte_buffer(buf)
|
|
|
|
assert events == [("message", '{"ok": true}')]
|
|
assert buf == bytearray()
|
|
|
|
|
|
def test_complete_event_with_invalid_utf8_raises_loud() -> None:
|
|
"""Invalid UTF-8 in a *complete* event surfaces loudly (not silent corruption)."""
|
|
# Build a complete event whose data field has invalid UTF-8 bytes
|
|
# NOT split across a chunk boundary — this is a true upstream bug
|
|
# we want operators to see, not silently fix-up.
|
|
bad = b'event: content_block_delta\ndata: {"text":"\xff\xfe"}\n\n'
|
|
buf = bytearray()
|
|
buf.extend(bad)
|
|
with pytest.raises(UnicodeDecodeError):
|
|
parse_sse_events_from_byte_buffer(buf)
|