mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
## Description `headroom proxy` can wedge when code-aware compression enters the tree-sitter Perl external scanner and the native scan keeps the GIL indefinitely. Current main still has two routes into that scanner: explicit `perl` or `pl` hints flow through `CodeAwareCompressor`, and `detect_language()` can nominate Perl on non-Perl code because its prefilter matches generic sigils such as decorators, JSDoc tags, and shell variables before phase 2 parses every surviving candidate grammar. This change quarantines Perl at the code-aware compression funnel without widening scope. Perl remains recognized at the input boundary, but live proxy compression no longer requests a Perl parser. Non-Perl code keeps its existing code-aware behavior. Real Perl falls back through the existing safe Kompress or passthrough contract instead of entering tree-sitter. The diff stays inside `headroom/transforms/code_compressor.py` plus focused parser-safety regressions. Refs #2185 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Added a Perl quarantine list in `headroom/transforms/code_compressor.py` and used it to stop Perl candidate parsing during language detection. - Added a hard `_get_parser()` guard so no live code-aware path can construct a Perl parser. - Routed resolved explicit Perl hints through the existing safe fallback or passthrough contract before AST compression. - Added focused parser-safety regressions for non-Perl candidate bleed, explicit `perl` and `pl`, inferred real Perl, mixed fenced Perl content, fallback-disabled passthrough, and non-Perl negative space. ## Testing - [x] Unit tests pass (`uv run pytest tests/test_perl_scanner_safety.py -q`) - [x] Linting passes (`uv run ruff check headroom/transforms/code_compressor.py tests/test_perl_scanner_safety.py`) - [ ] Type checking passes (`uv run mypy headroom`) - [x] New tests added for new functionality when applicable - [ ] Manual testing performed ### Test Output ```text uv run pytest tests/test_perl_scanner_safety.py -q 8 passed in 1.93s uv run ruff check headroom/transforms/code_compressor.py tests/test_perl_scanner_safety.py All checks passed! uv run ruff format headroom/transforms/code_compressor.py tests/test_perl_scanner_safety.py --check 2 files already formatted ``` ## Real Behavior Proof - Environment: Windows, Python from the synced `uv` environment, `dev` and `code` extras installed, no provider call - Exact command / steps: Run `uv run pytest tests/test_perl_scanner_safety.py -q`. - Observed result: `8 passed in 1.93s`; the suite proves explicit `perl` and `pl`, inferred real Perl, mixed fenced Perl, and non-Perl negative-space routes all avoid Perl parser entry. - Not tested: the reporter's macOS payload and long-running concurrent workload ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I have updated the CHANGELOG.md if applicable ## Additional Notes - Use `Refs #2185`, not `Closes #2185`. The wedge surface is this PR's scope, but #2185 also carries a separate orphaned `headroom mcp serve` report that this slice does not address. - This is a reachability fix. It does not repair the upstream Perl scanner and it does not harden other grammars against the same class of native wedge. - `CHANGELOG.md` remains unchanged because Headroom generates release notes from conventional commits. - Merged PR https://github.com/headroomlabs-ai/headroom/pull/2114 addresses a different cooperative compression stall and stays separate from this native parser-entry slice. --------- Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
173 lines
4.3 KiB
Python
173 lines
4.3 KiB
Python
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
import headroom.transforms.code_compressor as cc
|
|
from headroom.transforms.code_compressor import (
|
|
CodeAwareCompressor,
|
|
CodeCompressionResult,
|
|
CodeCompressorConfig,
|
|
CodeLanguage,
|
|
unload_tree_sitter,
|
|
)
|
|
from headroom.transforms.content_router import ContentRouter, ContentRouterConfig
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_tree_sitter():
|
|
unload_tree_sitter()
|
|
yield
|
|
unload_tree_sitter()
|
|
|
|
|
|
def _compressor(**overrides) -> CodeAwareCompressor:
|
|
defaults = {
|
|
"min_tokens_for_compression": 1,
|
|
"fallback_to_kompress": False,
|
|
"enable_ccr": False,
|
|
}
|
|
defaults.update(overrides)
|
|
return CodeAwareCompressor(CodeCompressorConfig(**defaults))
|
|
|
|
|
|
def _record_parser_calls(monkeypatch):
|
|
calls: list[str] = []
|
|
real_get_parser = cc._get_parser
|
|
|
|
def spy(language: str):
|
|
calls.append(language)
|
|
return real_get_parser(language)
|
|
|
|
monkeypatch.setattr(cc, "_get_parser", spy)
|
|
return calls
|
|
|
|
|
|
def test_typescript_with_decorators_stays_typescript_without_perl(monkeypatch):
|
|
tree_sitter_installed = cc._tree_sitter_importable()
|
|
calls = _record_parser_calls(monkeypatch)
|
|
code = """
|
|
import { Component, Input } from "@angular/core";
|
|
|
|
@Component({
|
|
selector: "demo-card",
|
|
template: "<div>{{ title }}</div>",
|
|
})
|
|
export class DemoCardComponent {
|
|
@Input() title: string = "";
|
|
|
|
render(items: string[]): string {
|
|
return items.map((item) => item.trim()).join(",");
|
|
}
|
|
}
|
|
""".strip()
|
|
|
|
result = _compressor().compress(code)
|
|
|
|
assert result.language == CodeLanguage.TYPESCRIPT
|
|
assert "perl" not in calls
|
|
if tree_sitter_installed:
|
|
assert "typescript" in calls
|
|
|
|
|
|
def test_explicit_perl_uses_existing_fallback_without_requesting_parser(monkeypatch):
|
|
calls = _record_parser_calls(monkeypatch)
|
|
compressor = _compressor(fallback_to_kompress=True)
|
|
code = "use strict;\nsub demo {\n my $value = shift;\n return $value;\n}\n"
|
|
sentinel = CodeCompressionResult(
|
|
compressed="kompress sentinel",
|
|
original=code,
|
|
original_tokens=10,
|
|
compressed_tokens=3,
|
|
compression_ratio=0.3,
|
|
language=CodeLanguage.UNKNOWN,
|
|
language_confidence=0.0,
|
|
syntax_valid=False,
|
|
)
|
|
|
|
with patch.object(compressor, "_fallback_compress", return_value=sentinel) as fallback:
|
|
result = compressor.compress(code, language="perl")
|
|
|
|
assert result is sentinel
|
|
fallback.assert_called_once()
|
|
assert "perl" not in calls
|
|
|
|
|
|
@pytest.mark.parametrize("hint", ["perl", "pl"])
|
|
def test_explicit_perl_passthrough_keeps_original_without_parser(monkeypatch, hint):
|
|
calls = _record_parser_calls(monkeypatch)
|
|
code = "use strict;\nsub demo {\n my $value = shift;\n return $value;\n}\n"
|
|
|
|
result = _compressor().compress(code, language=hint)
|
|
|
|
assert result.compressed == code
|
|
assert result.language == CodeLanguage.PERL
|
|
assert "perl" not in calls
|
|
|
|
|
|
def test_real_perl_detects_unknown_without_requesting_parser(monkeypatch):
|
|
calls = _record_parser_calls(monkeypatch)
|
|
code = """
|
|
package Demo;
|
|
use strict;
|
|
use warnings;
|
|
|
|
sub greet {
|
|
my ($name) = @_;
|
|
return "hello $name";
|
|
}
|
|
|
|
1;
|
|
""".strip()
|
|
|
|
result = _compressor().compress(code)
|
|
|
|
assert result.language == CodeLanguage.UNKNOWN
|
|
assert result.compressed == code
|
|
assert "perl" not in calls
|
|
|
|
|
|
def test_fenced_perl_router_path_never_requests_parser(monkeypatch):
|
|
calls = _record_parser_calls(monkeypatch)
|
|
router = ContentRouter(ContentRouterConfig(enable_code_aware=True, min_section_tokens=1))
|
|
content = """
|
|
```perl
|
|
use strict;
|
|
use warnings;
|
|
|
|
sub greet {
|
|
my ($name) = @_;
|
|
return "hello $name";
|
|
}
|
|
```
|
|
""".strip()
|
|
|
|
result = router.compress(content)
|
|
|
|
assert "```perl" in result.compressed
|
|
assert "perl" not in calls
|
|
|
|
|
|
def test_go_still_uses_non_perl_parser(monkeypatch):
|
|
tree_sitter_installed = cc._tree_sitter_importable()
|
|
calls = _record_parser_calls(monkeypatch)
|
|
code = """
|
|
package main
|
|
|
|
import "fmt"
|
|
|
|
func main() {
|
|
fmt.Println("hello")
|
|
}
|
|
""".strip()
|
|
|
|
result = _compressor().compress(code)
|
|
|
|
assert result.language == CodeLanguage.GO
|
|
assert "perl" not in calls
|
|
if tree_sitter_installed:
|
|
assert "go" in calls
|
|
|
|
|
|
def test_get_parser_refuses_quarantined_perl():
|
|
with pytest.raises(ValueError, match="quarantined"):
|
|
cc._get_parser("perl")
|