diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py index 149fc76b3..897318b5c 100644 --- a/headroom/proxy/handlers/anthropic.py +++ b/headroom/proxy/handlers/anthropic.py @@ -1385,6 +1385,7 @@ class AnthropicHandlerMixin: else "deferred:dropped" ] timing = {} + waste_signals = None result = _DeferredCompressionResult() else: diff --git a/headroom/transforms/code_compressor.py b/headroom/transforms/code_compressor.py index e9872d936..1023d1d12 100644 --- a/headroom/transforms/code_compressor.py +++ b/headroom/transforms/code_compressor.py @@ -102,6 +102,9 @@ def _tree_sitter_importable() -> bool: return False +_UNSAFE_TREE_SITTER_LANGUAGES: frozenset[str] = frozenset({"perl"}) + + def _get_parser(language: str) -> Any: """Get a tree-sitter parser for the given language. @@ -129,6 +132,8 @@ def _get_parser(language: str) -> Any: ImportError: If tree-sitter is not installed. ValueError: If language is not supported. """ + if language in _UNSAFE_TREE_SITTER_LANGUAGES: + raise ValueError(f"Language '{language}' is quarantined for code-aware compression.") # NOTE: guard on importability (not _check_tree_sitter_available), because # _check_tree_sitter_available now performs a real end-to-end parse via # _get_parser; guarding on it here would recurse. @@ -161,7 +166,7 @@ def _get_parser(language: str) -> Any: except Exception as e: raise ValueError( f"Language '{language}' is not supported by tree-sitter. " - f"Supported: python, javascript, typescript, go, rust, java, c, cpp, csharp, perl. " + f"Supported: python, javascript, typescript, go, rust, java, c, cpp, csharp. " f"Error: {e}" ) from e @@ -714,6 +719,16 @@ def detect_language(code: str) -> tuple[CodeLanguage, float]: if candidates[CodeLanguage.CPP] >= 2: candidates[CodeLanguage.C] = 0 + perl_score = candidates.get(CodeLanguage.PERL, 0) + if perl_score > 0: + best_non_perl = max( + (score for lang, score in candidates.items() if lang != CodeLanguage.PERL), + default=0, + ) + if perl_score > best_non_perl: + return CodeLanguage.UNKNOWN, 0.0 + candidates.pop(CodeLanguage.PERL, None) + # Phase 2: If tree-sitter available, parse with candidates and pick fewest errors if _check_tree_sitter_available(): best_lang = CodeLanguage.UNKNOWN @@ -1142,6 +1157,19 @@ class CodeAwareCompressor(Transform): language_confidence=0.0, syntax_valid=True, ) + if detected_lang == CodeLanguage.PERL: + if self.config.fallback_to_kompress: + return self._fallback_compress(code, original_tokens) + return CodeCompressionResult( + compressed=code, + original=code, + original_tokens=original_tokens, + compressed_tokens=original_tokens, + compression_ratio=1.0, + language=detected_lang, + language_confidence=confidence, + syntax_valid=True, + ) # Check if tree-sitter is available if not _check_tree_sitter_available(): diff --git a/server.json b/server.json index c660bf2fe..20fa7726d 100644 --- a/server.json +++ b/server.json @@ -9,13 +9,13 @@ "source": "github", "id": "1129940957" }, - "version": "0.27.0", + "version": "0.31.0", "packages": [ { "registryType": "pypi", "registryBaseUrl": "https://pypi.org", "identifier": "headroom-ai", - "version": "0.27.0", + "version": "0.31.0", "runtimeHint": "uvx", "runtimeArguments": [ { diff --git a/tests/test_cli/test_wrap_zcode.py b/tests/test_cli/test_wrap_zcode.py index be20cf9f4..17b7a18ad 100644 --- a/tests/test_cli/test_wrap_zcode.py +++ b/tests/test_cli/test_wrap_zcode.py @@ -121,7 +121,7 @@ def test_wrap_prints_proxy_urls( def fake_watcher(**kwargs): # noqa: ANN003 print_fn = kwargs.get("print_setup_lines") if callable(print_fn): - print_fn() + print_fn(kwargs["port"]) with patch.object(wrap_mod, "_ensure_rtk_binary", return_value=fake_rtk): with patch.object(wrap_mod, "_run_proxy_only_watcher", side_effect=fake_watcher): diff --git a/tests/test_cold_start_fast_pass.py b/tests/test_cold_start_fast_pass.py index c90e915aa..6c6baa4c6 100644 --- a/tests/test_cold_start_fast_pass.py +++ b/tests/test_cold_start_fast_pass.py @@ -38,6 +38,9 @@ class _DummyMetrics: async def record_failed(self, **kwargs): return None + def record_compression_failed(self, reason: str) -> None: + return None + async def record_rate_limited(self, **kwargs): return None diff --git a/tests/test_perl_scanner_safety.py b/tests/test_perl_scanner_safety.py new file mode 100644 index 000000000..ecce4b5c3 --- /dev/null +++ b/tests/test_perl_scanner_safety.py @@ -0,0 +1,173 @@ +from unittest.mock import patch + +import pytest + +import headroom.transforms.code_compressor as cc +from headroom.transforms.code_compressor import ( + CodeAwareCompressor, + CodeCompressionResult, + CodeCompressorConfig, + CodeLanguage, + unload_tree_sitter, +) +from headroom.transforms.content_router import ContentRouter, ContentRouterConfig + + +@pytest.fixture(autouse=True) +def _reset_tree_sitter(): + unload_tree_sitter() + yield + unload_tree_sitter() + + +def _compressor(**overrides) -> CodeAwareCompressor: + defaults = { + "min_tokens_for_compression": 1, + "fallback_to_kompress": False, + "enable_ccr": False, + } + defaults.update(overrides) + return CodeAwareCompressor(CodeCompressorConfig(**defaults)) + + +def _record_parser_calls(monkeypatch): + calls: list[str] = [] + real_get_parser = cc._get_parser + + def spy(language: str): + calls.append(language) + return real_get_parser(language) + + monkeypatch.setattr(cc, "_get_parser", spy) + return calls + + +def test_typescript_with_decorators_stays_typescript_without_perl(monkeypatch): + tree_sitter_installed = cc._tree_sitter_importable() + calls = _record_parser_calls(monkeypatch) + code = """ +import { Component, Input } from "@angular/core"; + +@Component({ + selector: "demo-card", + template: "