From bdcfc322da0c4cde69931d641cfa18c76ddb138b Mon Sep 17 00:00:00 2001
From: Mubashir R <112580905+Mubashirrrr@users.noreply.github.com>
Date: Thu, 4 Jun 2026 00:32:56 +0500
Subject: [PATCH] fix: ignore brackets inside JSON strings when splitting mixed
 content (#553)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

_extract_json_block() counted raw [ ] { } per line via str.count() to
find where a JSON block ends. Any bracket/brace inside a JSON string
value (e.g. the "]" in {"path": "a]b"}) was counted as structural, so
the running balance hit zero early and the block was cut mid-array.

In ContentRouter._compress_mixed() this fragments one JSON array into
multiple sections: the array is truncated, a non-array fragment gets
mislabeled JSON_ARRAY, and the trailing "]" leaks into the next prose
section — so content is routed to the wrong compressor.

Walk the characters with a small in-string/escape state machine and
only count brackets/braces that are outside string literals. Behavior
is unchanged for JSON without brackets-in-strings.

Regression tests in tests/test_transforms_content_router.py cover both
the helper (_extract_json_block) and the end-to-end split
(split_into_sections). They fail before this change and pass after.

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
---
 headroom/transforms/content_router.py   | 29 +++++++++++-
 tests/test_transforms_content_router.py | 60 +++++++++++++++++++++++++
 2 files changed, 87 insertions(+), 2 deletions(-)

diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 8c02c7753..6dc6f0203 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -658,13 +658,38 @@ def _extract_json_block(lines: list[str], start: int) -> tuple[str | None, int]:
     bracket_count = 0
     brace_count = 0
     json_lines = []
+    in_string = False
+    escaped = False
 
     for i in range(start, len(lines)):
         line = lines[i]
         json_lines.append(line)
 
-        bracket_count += line.count("[") - line.count("]")
-        brace_count += line.count("{") - line.count("}")
+        # Count brackets/braces, but ignore any that appear inside a JSON
+        # string literal — a naive line.count() treats e.g. the "]" in
+        # {"path": "a]b"} as a closing bracket and terminates the block
+        # early, splitting one array across multiple sections.
+        for ch in line:
+            if escaped:
+                escaped = False
+                continue
+            if ch == "\\":
+                if in_string:
+                    escaped = True
+                continue
+            if ch == '"':
+                in_string = not in_string
+                continue
+            if in_string:
+                continue
+            if ch == "[":
+                bracket_count += 1
+            elif ch == "]":
+                bracket_count -= 1
+            elif ch == "{":
+                brace_count += 1
+            elif ch == "}":
+                brace_count -= 1
 
         if bracket_count <= 0 and brace_count <= 0 and json_lines:
             return "\n".join(json_lines), i
diff --git a/tests/test_transforms_content_router.py b/tests/test_transforms_content_router.py
index 8ee570979..01c3732c6 100644
--- a/tests/test_transforms_content_router.py
+++ b/tests/test_transforms_content_router.py
@@ -171,6 +171,66 @@ def test_mixed_content_section_splitting_and_json_extraction() -> None:
     assert _extract_json_block(["{", '"a": 1'], 0) == (None, 0)
 
 
+def test_extract_json_block_ignores_brackets_inside_strings() -> None:
+    """Brackets/braces inside JSON string values must not end the block early.
+
+    Regression: counting raw ``[``/``]``/``{``/``}`` per line treated the
+    ``]`` inside ``{"path": "a]b"}`` as a closing bracket, so the array was
+    truncated mid-way and the remaining rows leaked into later sections.
+    """
+    import json as _json
+
+    lines = [
+        "[",
+        '  {"path": "a]b"},',
+        '  {"path": "c"}',
+        "]",
+    ]
+    block, end_idx = _extract_json_block(lines, 0)
+    assert end_idx == 3
+    assert block is not None
+    parsed = _json.loads(block)
+    assert parsed == [{"path": "a]b"}, {"path": "c"}]
+
+    # Braces inside a string value must likewise be ignored.
+    obj_lines = [
+        "{",
+        '  "msg": "use {curly} and [square]",',
+        '  "n": 1',
+        "}",
+    ]
+    obj_block, obj_end = _extract_json_block(obj_lines, 0)
+    assert obj_end == 3
+    assert obj_block is not None
+    assert _json.loads(obj_block) == {"msg": "use {curly} and [square]", "n": 1}
+
+
+def test_split_into_sections_keeps_json_array_with_bracket_in_string() -> None:
+    """A JSON array embedded in prose stays one JSON section, not fragments.
+
+    With the bracket-in-string bug, the array below split into a truncated
+    JSON section plus a stray ``]`` glued onto the trailing prose.
+    """
+    import json as _json
+
+    content = "\n".join(
+        [
+            "prose line here that is long enough to matter",
+            "[",
+            '  {"path": "a]b"},',
+            '  {"path": "c"}',
+            "]",
+            "trailing prose",
+        ]
+    )
+
+    sections = split_into_sections(content)
+    json_sections = [s for s in sections if s.content_type == ContentType.JSON_ARRAY]
+    assert len(json_sections) == 1
+    parsed = _json.loads(json_sections[0].content)
+    assert parsed == [{"path": "a]b"}, {"path": "c"}]
+
+
 def test_content_router_strategy_and_compress_paths(monkeypatch: pytest.MonkeyPatch) -> None:
     router = ContentRouter(ContentRouterConfig(prefer_code_aware_for_code=False))