From f39858c23325f9f27b47a738731e7260f7b59d9e Mon Sep 17 00:00:00 2001 From: Veesh Goldman Date: Tue, 23 Jun 2026 02:47:41 +0300 Subject: [PATCH] feat(code): add Perl support to code-aware compressor (#1125) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Adds Perl as a supported language for `CodeAwareCompressor` / `CodeStructureHandler`. Function bodies are compressed while `use`/`require` imports, `sub`/`method` signatures, and `package`/`class`/`role` declarations are preserved — bringing Perl up to parity with the other Tier-2 languages. No new dependencies: the Perl grammar already ships in `tree-sitter-language-pack` (already a Headroom dependency), so this is pure configuration. Closes # ## Type of Change - [ ] Bug fix (non-breaking change that fixes an issue) - [x] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - `code_handler.py`: Perl entries in the four per-language tables — `_STRUCTURAL_NODE_TYPES`, `_SIGNATURE_PATTERNS` (regex fallback), `_LANGUAGE_MARKERS` (detection), `_IMPORT_PATTERNS`. The existing `_CONTAINER_BODY_TYPES` already covers Perl's `block` body node, so no change was needed there. - `code_compressor.py`: `CodeLanguage.PERL` enum value, a data-driven `LangConfig`, a `_LANGUAGE_PREFILTER` entry, and the supported-language string in the parser error message. - Node-type names (`subroutine_declaration_statement`, `package_statement`, `signature`, `block`, …) are from the `tree-sitter-perl/tree-sitter-perl` grammar (MIT). - Tests: 1 detection test + 2 regex-path signature/import-preservation tests. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [x] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text $ pytest tests/test_compression/test_code_handler.py -q collected 28 items tests/test_compression/test_code_handler.py ............................ [100%] ======================== 28 passed, 1 warning in 2.53s ========================= $ ruff check headroom/compression/handlers/code_handler.py headroom/transforms/code_compressor.py tests/test_compression/test_code_handler.py All checks passed! $ mypy headroom/compression/handlers/code_handler.py headroom/transforms/code_compressor.py Success: no issues found in 2 source files ``` ## Real Behavior Proof - Environment: Python 3.12, `pip install -e ".[code]"` (tree-sitter-language-pack installed, `is_tree_sitter_available() == True`). - Exact command / steps: ran `CodeStructureHandler().get_mask(code, language="perl")` on a real Perl module (package + two subs with bodies). - Observed result: detected as `perl`, parsed via the `tree-sitter` path (not regex), and the preserved span was exactly the imports + package + sub signatures, with both sub bodies marked compressible: ```text tree-sitter available: True parser: tree-sitter | detected: perl --- PRESERVED (signatures/imports/structure) --- use strict;use warnings;package Greeter;sub new sub greet ``` - Not tested: the full proxy/MCP server end-to-end path (out of scope — this PR only touches the code compressor's language tables). ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] I have updated the CHANGELOG.md if applicable ## Additional Notes - Docs/CHANGELOG left unchecked — happy to add a Perl line to either if you'd like; I wasn't sure of your preferred location. - *Disclosure: I maintain the upstream `tree-sitter-perl` grammar this relies on. It's already a transitive dependency of Headroom via `tree-sitter-language-pack` — this PR only adds config to use it, with no dependency changes.* Co-authored-by: Claude Opus 4.8 (1M context) --- headroom/compression/handlers/code_handler.py | 15 +++++++++++++ headroom/transforms/code_compressor.py | 22 ++++++++++++++++++- tests/test_compression/test_code_handler.py | 16 ++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/headroom/compression/handlers/code_handler.py b/headroom/compression/handlers/code_handler.py index a15466e65..7066e511d 100644 --- a/headroom/compression/handlers/code_handler.py +++ b/headroom/compression/handlers/code_handler.py @@ -173,6 +173,15 @@ _STRUCTURAL_NODE_TYPES: dict[str, set[str]] = { "interface_declaration", "annotation", }, + "perl": { + "use_statement", + "use_version_statement", + "subroutine_declaration_statement", + "method_declaration_statement", + "package_statement", + "class_statement", + "role_statement", + }, } # Regex patterns for fallback detection @@ -211,6 +220,10 @@ _SIGNATURE_PATTERNS: dict[str, list[re.Pattern[str]]] = { re.compile(r"^\s*(public\s+)?(class|interface|enum)\s+\w+", re.MULTILINE), re.compile(r"^\s*@\w+(\([^)]*\))?\s*$", re.MULTILINE), ], + "perl": [ + re.compile(r"^\s*sub\s+\w+\s*(\([^)]*\))?", re.MULTILINE), + re.compile(r"^\s*(package|class|role)\s+[\w:]+", re.MULTILINE), + ], } # Body child node types for container definitions (classes, impls, @@ -238,6 +251,7 @@ _LANGUAGE_MARKERS: dict[str, list[str]] = { "go": ["func ", "package ", "import (", "type "], "rust": ["fn ", "let mut", "impl ", "pub fn", "use "], "java": ["public class", "private ", "protected ", "void "], + "perl": ["sub ", "my $", "our $", "package ", "use strict"], } # Import patterns for fallback @@ -248,6 +262,7 @@ _IMPORT_PATTERNS: dict[str, re.Pattern[str]] = { "go": re.compile(r'^\s*import\s+(\(|")', re.MULTILINE), "rust": re.compile(r"^\s*use\s+\w+", re.MULTILINE), "java": re.compile(r"^\s*import\s+[\w.]+;", re.MULTILINE), + "perl": re.compile(r"^\s*(use|require)\s+[\w:]+", re.MULTILINE), } diff --git a/headroom/transforms/code_compressor.py b/headroom/transforms/code_compressor.py index db2cd78b6..984661d34 100644 --- a/headroom/transforms/code_compressor.py +++ b/headroom/transforms/code_compressor.py @@ -129,7 +129,7 @@ def _get_parser(language: str) -> Any: except Exception as e: raise ValueError( f"Language '{language}' is not supported by tree-sitter. " - f"Supported: python, javascript, typescript, go, rust, java, c, cpp. " + f"Supported: python, javascript, typescript, go, rust, java, c, cpp, perl. " f"Error: {e}" ) from e @@ -183,6 +183,7 @@ class CodeLanguage(Enum): JAVA = "java" C = "c" CPP = "cpp" + PERL = "perl" UNKNOWN = "unknown" @@ -317,6 +318,20 @@ _LANG_CONFIGS: dict[CodeLanguage, LangConfig] = { uses_colon_after_signature=False, detection_hints=("#include", "namespace ", "class ", "::"), ), + CodeLanguage.PERL: LangConfig( + import_nodes=frozenset({"use_statement", "use_version_statement"}), + function_nodes=frozenset( + {"subroutine_declaration_statement", "method_declaration_statement"} + ), + class_nodes=frozenset({"package_statement", "class_statement", "role_statement"}), + type_nodes=frozenset(), + body_node_types=frozenset({"block"}), + decorator_node=None, + comment_prefix="#", + uses_colon_after_signature=False, + package_node="package_statement", + detection_hints=("sub ", "my ", "our ", "use ", "package "), + ), } @@ -506,6 +521,11 @@ _LANGUAGE_PREFILTER: dict[CodeLanguage, list[re.Pattern[str]]] = { re.compile(r"\bnamespace\s+\w+", re.MULTILINE), re.compile(r"::\w+", re.MULTILINE), ], + CodeLanguage.PERL: [ + re.compile(r"^\s*(sub|package|use|require)\s+[\w:]+", re.MULTILINE), + re.compile(r"^\s*(my|our|local)\s+[\$@%]", re.MULTILINE), + re.compile(r"[\$@%]\w+", re.MULTILINE), + ], } diff --git a/tests/test_compression/test_code_handler.py b/tests/test_compression/test_code_handler.py index 4cebc697e..3987ed616 100644 --- a/tests/test_compression/test_code_handler.py +++ b/tests/test_compression/test_code_handler.py @@ -70,6 +70,10 @@ class TestLanguageDetection: code = "use std::io;\n\npub fn main() {\n let mut x = 1;\n}\n" assert handler._detect_language(code) == "rust" + def test_detects_perl(self, handler): + code = "use strict;\npackage Foo;\n\nsub greet {\n my $name = shift;\n return $name;\n}\n" + assert handler._detect_language(code) == "perl" + def test_falls_back_to_default(self): handler = CodeStructureHandler(default_language="javascript") assert handler._detect_language("plain words only here") == "javascript" @@ -110,6 +114,18 @@ class TestRegexFallbackLanguages: start = code.index(sig) assert all(result.mask.mask[i] for i in range(start, start + len(sig))) + def test_perl_sub_signature_preserved(self, handler): + code = "sub add {\n my ($a, $b) = @_;\n return $a + $b;\n}\n" + result = handler.get_mask(code, language="perl") + sig = "sub add" + start = code.index(sig) + assert all(result.mask.mask[i] for i in range(start, start + len(sig))) + + def test_perl_use_import_preserved(self, handler): + code = "use strict;\nuse warnings;\n\nmy $x = 1;\n" + result = handler.get_mask(code, language="perl") + assert all(result.mask.mask[i] for i in range(len("use strict"))) + def test_regex_confidence_lower_than_tree_sitter(self, handler): result = handler.get_mask("def f():\n pass\n", language="python") assert result.confidence == 0.7