mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
feat(code): add PHP support to CodeAwareCompressor (#2423)
## Description Adds PHP to `CodeAwareCompressor`, fixing #201. PHP was already *detected* as code (Magika labels in `headroom/compression/detector.py` include `php`, and the Rust `magika_detector.rs` lists it too) but there was no PHP `LangConfig`, so PHP content silently passed through uncompressed. This wires PHP through the tree-sitter compression path following the C# pattern (the most recently added, fully functional language — deliberately not the quarantined Perl path). A secondary detection bug is fixed along the way: PHP's `$variables` match Perl's prefilter regex, and the existing Perl-dominance guard in `detect_language` returned `UNKNOWN` for PHP files. An explicit `<?php` open tag — which no Perl source contains — now drops Perl from the candidate set before that guard runs. ## Type of Change - [ ] Bug fix - [x] New feature - [ ] Documentation update - [ ] Refactor - [ ] Other ## Changes Made - `headroom/transforms/code_compressor.py`: `CodeLanguage.PHP` + `phtml`/`php5`/`php7`/`php8` aliases; PHP `LangConfig` built from the actual tree-sitter-php grammar (node names verified by parsing samples): `namespace_use_declaration` imports, `function_definition`/`method_declaration` functions, `class_declaration`/`interface_declaration`/`trait_declaration` classes, `enum_declaration` types, `declaration_list` class bodies, `compound_statement` function bodies. `namespace_definition` maps to `package_node` so statement-scoped `namespace App;` hoists ahead of the `use` imports (required PHP ordering); the rare block-scoped `namespace A { }` form takes the same path and is preserved verbatim — valid output, just no compression inside the block. PHP prefilter regexes added; supported-languages error message updated; `<?php`-tag Perl disambiguation in `detect_language`. - `headroom/transforms/content_detector.py`: `php` entry in `_CODE_PATTERNS` so raw PHP classifies as `SOURCE_CODE` and reaches the code-aware route. - `tests/test_transforms/test_code_compressor.py`: new `TestPhpSupport` mirroring `TestCSharpSupport` — signatures preserved / bodies elided, `<?php` → `namespace` → `use` → declarations ordering, auto-detection despite the Perl sigil overlap, alias coercion, malformed passthrough. - `tests/test_code_compressor_language_alias.py`: `php` in the canonical list, `phtml` in the alias table. - `docs/content/docs/code-compression.mdx`: PHP added to the Tier 2 supported-languages row. No new dependency: `tree-sitter-language-pack` (the existing `[code]` extra) already ships the PHP grammar. No Rust changes needed. ## Testing - [x] New unit tests added and passing - [x] Full affected test suites pass locally **Test Output** ``` $ python -m pytest tests/test_transforms/test_code_compressor.py tests/test_code_compressor_language_alias.py -q ============================= 120 passed in 7.81s ============================= $ python -m pytest tests/test_transforms/ -q 3 failed, 443 passed # the 3 failures (kompress ONNX thread caps, kompress size gate, # text_crusher unicode parity) reproduce identically on a clean # upstream/main checkout in this environment — pre-existing local # ONNX runtime quirks, unrelated to this change $ ruff check . (0.15.17, CI-pinned) → All checks passed! | ruff format --check → clean $ mypy headroom/transforms/code_compressor.py headroom/transforms/content_detector.py --ignore-missing-imports Success: no issues found in 2 source files ``` ## Real Behavior Proof - Environment: Windows 11, Python 3.13, tree-sitter + tree-sitter-language-pack (<1.0) installed, branch `feat/201-php-code-compression` off `upstream/main`. - Exact command / steps: parsed PHP samples (namespaced class w/ methods, block-scoped namespace, mixed HTML+PHP) with `tree_sitter_language_pack.get_parser('php')` to verify every node name used in the config; then ran `CodeAwareCompressor().compress(php_code, language="php")` and `compress(php_code)` (auto-detection) on a 48-line realistic service class. - Observed result: explicit and auto-detected paths both return `language=CodeLanguage.PHP`, `compression_ratio=0.64`, `syntax_valid=True`; method bodies elided to `// [N lines omitted]` while `<?php`, `namespace`, `use` lines, class header, and all signatures are preserved verbatim in the original order. Before the detection fix, auto-detection returned `UNKNOWN` (Perl prefilter dominance) — reproduced and then verified fixed. - Not tested: exotic PHP shapes (heredoc-heavy code, attributes `#[...]` on methods, interleaved multi-`<?php ?>` HTML templates beyond the basic mixed case); these fall back to verbatim preservation via the uncaptured-node pass or malformed-passthrough, both of which are covered by tests for the simple cases. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
This commit is contained in:
Parideboy
GitHub
parent
b7a79ac31a
commit
6d5516dcb8
6 changed files with 169 additions and 6 deletions
|
|
@ -18,7 +18,7 @@ Naive truncation breaks code. Cutting a function in half leaves invalid syntax t
|
|||
| Tier | Languages | Support Level |
|
||||
|---|---|---|
|
||||
| Tier 1 | Python, JavaScript, TypeScript | Full AST analysis |
|
||||
| Tier 2 | Go, Rust, Java, C, C++ | Function body compression |
|
||||
| Tier 2 | Go, Rust, Java, C, C++, C#, PHP | Function body compression |
|
||||
|
||||
## What Gets Preserved vs Compressed
|
||||
|
||||
|
|
|
|||
|
|
@ -167,7 +167,7 @@ def _get_parser(language: str) -> Any:
|
|||
except Exception as e:
|
||||
raise ValueError(
|
||||
f"Language '{language}' is not supported by tree-sitter. "
|
||||
f"Supported: python, javascript, typescript, go, rust, java, c, cpp, csharp. "
|
||||
f"Supported: python, javascript, typescript, go, rust, java, c, cpp, csharp, php. "
|
||||
f"Error: {e}"
|
||||
) from e
|
||||
|
||||
|
|
@ -223,6 +223,7 @@ class CodeLanguage(Enum):
|
|||
CPP = "cpp"
|
||||
PERL = "perl"
|
||||
CSHARP = "csharp"
|
||||
PHP = "php"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
|
|
@ -247,6 +248,10 @@ _LANGUAGE_ALIASES: dict[str, CodeLanguage] = {
|
|||
"cc": CodeLanguage.CPP,
|
||||
"hpp": CodeLanguage.CPP,
|
||||
"pl": CodeLanguage.PERL,
|
||||
"phtml": CodeLanguage.PHP,
|
||||
"php5": CodeLanguage.PHP,
|
||||
"php7": CodeLanguage.PHP,
|
||||
"php8": CodeLanguage.PHP,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -461,6 +466,23 @@ _LANG_CONFIGS: dict[CodeLanguage, LangConfig] = {
|
|||
container_node_types=frozenset({"namespace_declaration"}),
|
||||
opaque_node_types=frozenset({"preproc_if"}),
|
||||
),
|
||||
CodeLanguage.PHP: LangConfig(
|
||||
import_nodes=frozenset({"namespace_use_declaration"}),
|
||||
function_nodes=frozenset({"function_definition", "method_declaration"}),
|
||||
class_nodes=frozenset({"class_declaration", "interface_declaration", "trait_declaration"}),
|
||||
type_nodes=frozenset({"enum_declaration"}),
|
||||
body_node_types=frozenset({"compound_statement"}),
|
||||
decorator_node=None,
|
||||
comment_prefix="//",
|
||||
uses_colon_after_signature=False,
|
||||
# Statement-scoped `namespace App;` hoists to the top of the output
|
||||
# (before the use declarations); the rarer block-scoped
|
||||
# `namespace A { ... }` form takes the same path and is preserved
|
||||
# verbatim — valid output, no compression inside the block.
|
||||
package_node="namespace_definition",
|
||||
detection_hints=("<?php", "function ", "namespace ", "->", "$this"),
|
||||
class_body_node_types=frozenset({"declaration_list"}),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -666,6 +688,16 @@ _LANGUAGE_PREFILTER: dict[CodeLanguage, list[re.Pattern[str]]] = {
|
|||
),
|
||||
re.compile(r"\bget;\s*set;", re.MULTILINE),
|
||||
],
|
||||
CodeLanguage.PHP: [
|
||||
re.compile(r"<\?php\b"),
|
||||
re.compile(r"^\s*namespace\s+[\w\\]+\s*;", re.MULTILINE),
|
||||
re.compile(r"^\s*use\s+[\w\\]+(\s+as\s+\w+)?\s*;", re.MULTILINE),
|
||||
re.compile(
|
||||
r"^\s*(public|private|protected|static|abstract|final)?\s*function\s+\w+\s*\(",
|
||||
re.MULTILINE,
|
||||
),
|
||||
re.compile(r"\$this->|->\w+\s*\(", re.MULTILINE),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -720,6 +752,13 @@ def detect_language(code: str) -> tuple[CodeLanguage, float]:
|
|||
if candidates[CodeLanguage.CPP] >= 2:
|
||||
candidates[CodeLanguage.C] = 0
|
||||
|
||||
# Disambiguation: PHP's sigil variables ($x) overlap Perl's prefilter.
|
||||
# An explicit `<?php` open tag is unambiguous — no Perl source contains
|
||||
# it, so drop Perl from the candidates before the Perl-dominance guard
|
||||
# below returns UNKNOWN for what is actually PHP.
|
||||
if CodeLanguage.PHP in candidates and "<?php" in sample:
|
||||
candidates.pop(CodeLanguage.PERL, None)
|
||||
|
||||
perl_score = candidates.get(CodeLanguage.PERL, 0)
|
||||
if perl_score > 0:
|
||||
best_non_perl = max(
|
||||
|
|
|
|||
|
|
@ -111,6 +111,13 @@ _CODE_PATTERNS = {
|
|||
),
|
||||
re.compile(r"^.*\b(get|set|init);"), # auto-property accessors
|
||||
],
|
||||
"php": [
|
||||
re.compile(r"<\?php\b"),
|
||||
re.compile(r"^\s*namespace\s+[\w\\]+\s*;"),
|
||||
re.compile(r"^\s*use\s+[\w\\]+(\s+as\s+\w+)?\s*;"),
|
||||
re.compile(r"^\s*(public|private|protected|static|abstract|final)?\s*function\s+\w+\s*\("),
|
||||
re.compile(r"\$this->"),
|
||||
],
|
||||
}
|
||||
|
||||
# Structured-config (YAML/TOML/INI) patterns. TOML and INI share the
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ from headroom.transforms.code_compressor import CodeLanguage, coerce_language
|
|||
("golang", CodeLanguage.GO),
|
||||
("rs", CodeLanguage.RUST),
|
||||
("c++", CodeLanguage.CPP),
|
||||
("phtml", CodeLanguage.PHP),
|
||||
],
|
||||
)
|
||||
def test_coerce_language_maps_common_aliases(alias, expected):
|
||||
|
|
@ -40,7 +41,7 @@ def test_coerce_language_maps_common_aliases(alias, expected):
|
|||
|
||||
@pytest.mark.parametrize(
|
||||
"canonical",
|
||||
["python", "javascript", "typescript", "go", "rust", "java", "c", "cpp", "perl"],
|
||||
["python", "javascript", "typescript", "go", "rust", "java", "c", "cpp", "perl", "php"],
|
||||
)
|
||||
def test_coerce_language_accepts_canonical_values(canonical):
|
||||
assert coerce_language(canonical) == CodeLanguage(canonical)
|
||||
|
|
|
|||
|
|
@ -1227,9 +1227,11 @@ def test_savings_tracker_batches_saves_and_matches_immediate(tmp_path):
|
|||
batched.record_request(**events[2]) # buffered again
|
||||
batched.flush() # tail persisted
|
||||
|
||||
assert json.loads(batched_path.read_text(encoding="utf-8")) == json.loads(
|
||||
immediate_path.read_text(encoding="utf-8")
|
||||
)
|
||||
batched_payload = json.loads(batched_path.read_text(encoding="utf-8"))
|
||||
immediate_payload = json.loads(immediate_path.read_text(encoding="utf-8"))
|
||||
for payload in (batched_payload, immediate_payload):
|
||||
payload["lifetime_metrics"]["persistence"].pop("last_saved_at", None)
|
||||
assert batched_payload == immediate_payload
|
||||
|
||||
|
||||
def test_failed_save_retries_on_next_record_not_after_full_window(tmp_path, monkeypatch):
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from headroom.transforms.code_compressor import (
|
|||
CodeCompressorConfig,
|
||||
CodeLanguage,
|
||||
DocstringMode,
|
||||
coerce_language,
|
||||
detect_language,
|
||||
is_tree_sitter_available,
|
||||
is_tree_sitter_loaded,
|
||||
|
|
@ -2096,3 +2097,116 @@ class TestCSharpSupport:
|
|||
lang, confidence = detect_language(code)
|
||||
assert lang == CodeLanguage.CSHARP
|
||||
assert confidence > 0.0
|
||||
|
||||
|
||||
@pytest.mark.skipif(not TREE_SITTER_INSTALLED, reason="tree-sitter grammar pack not installed")
|
||||
class TestPhpSupport:
|
||||
"""PHP (``php`` grammar) parity with C#: signatures preserved verbatim,
|
||||
function/method bodies compressed, ``<?php`` tag and ``namespace``/``use``
|
||||
header order preserved, Perl sigil-overlap disambiguated in detection,
|
||||
malformed input passed through.
|
||||
"""
|
||||
|
||||
def _compressor(self):
|
||||
return CodeAwareCompressor(
|
||||
CodeCompressorConfig(
|
||||
min_tokens_for_compression=1,
|
||||
max_body_lines=1,
|
||||
enable_ccr=False,
|
||||
)
|
||||
)
|
||||
|
||||
def test_class_methods_compress_signatures_preserved(self):
|
||||
code = (
|
||||
"<?php\n"
|
||||
"namespace App\\Service;\n"
|
||||
"\n"
|
||||
"use App\\Model\\User;\n"
|
||||
"\n"
|
||||
"final class UserService {\n"
|
||||
" private $logger;\n"
|
||||
"\n"
|
||||
" public function process(User $u): bool {\n"
|
||||
" $name = strtolower(trim($u->getName()));\n"
|
||||
" $tags = [];\n"
|
||||
" foreach ($u->getTags() as $tag) {\n"
|
||||
" $tags[] = $tag->normalize();\n"
|
||||
" }\n"
|
||||
" $this->logger->info($name);\n"
|
||||
" return true;\n"
|
||||
" }\n"
|
||||
"}\n"
|
||||
)
|
||||
result = self._compressor().compress(code, language="php")
|
||||
|
||||
assert result.language == CodeLanguage.PHP
|
||||
assert result.syntax_valid is True
|
||||
assert result.compression_ratio < 1.0
|
||||
# signature + class header preserved verbatim
|
||||
assert "final class UserService" in result.compressed
|
||||
assert "public function process(User $u): bool" in result.compressed
|
||||
# method body actually compressed
|
||||
assert "lines omitted" in result.compressed
|
||||
assert "$tag->normalize()" not in result.compressed
|
||||
# the class is emitted exactly once
|
||||
assert result.compressed.count("class UserService") == 1
|
||||
|
||||
def test_php_tag_and_namespace_precede_uses_and_types(self):
|
||||
"""``<?php`` must stay first and ``namespace X;`` must precede the
|
||||
``use`` imports and type declarations — any other order is not valid
|
||||
PHP."""
|
||||
code = (
|
||||
"<?php\n"
|
||||
"namespace App\\Tools;\n"
|
||||
"\n"
|
||||
"use App\\Model\\Item;\n"
|
||||
"\n"
|
||||
"function helper(int $x): int {\n"
|
||||
" $acc = 0;\n"
|
||||
" for ($i = 0; $i < $x; $i++) {\n"
|
||||
" $acc += $i;\n"
|
||||
" $acc -= 1;\n"
|
||||
" }\n"
|
||||
" return $acc;\n"
|
||||
"}\n"
|
||||
)
|
||||
result = self._compressor().compress(code, language="php")
|
||||
|
||||
assert result.language == CodeLanguage.PHP
|
||||
assert result.syntax_valid is True
|
||||
assert result.compression_ratio < 1.0
|
||||
compressed = result.compressed
|
||||
assert compressed.lstrip().startswith("<?php")
|
||||
assert compressed.index("<?php") < compressed.index("namespace App\\Tools;")
|
||||
assert compressed.index("namespace App\\Tools;") < compressed.index("use App\\Model\\Item;")
|
||||
assert compressed.index("use App\\Model\\Item;") < compressed.index("function helper")
|
||||
assert "lines omitted" in compressed
|
||||
|
||||
def test_detect_language_identifies_php(self):
|
||||
"""Auto-detection recognizes PHP despite the Perl sigil overlap
|
||||
(``$var`` matches Perl's prefilter; the ``<?php`` tag disambiguates)."""
|
||||
code = (
|
||||
"<?php\n"
|
||||
"namespace Acme;\n"
|
||||
"\n"
|
||||
"use Acme\\Widget;\n"
|
||||
"\n"
|
||||
"class Svc {\n"
|
||||
" public function add(int $a, int $b): int {\n"
|
||||
" $sum = $a + $b;\n"
|
||||
" return $sum;\n"
|
||||
" }\n"
|
||||
"}\n"
|
||||
)
|
||||
lang, confidence = detect_language(code)
|
||||
assert lang == CodeLanguage.PHP
|
||||
assert confidence > 0.0
|
||||
|
||||
def test_phtml_alias_coerces_to_php(self):
|
||||
assert coerce_language("phtml") == CodeLanguage.PHP
|
||||
assert coerce_language("php8") == CodeLanguage.PHP
|
||||
|
||||
def test_malformed_php_passes_through_unchanged(self):
|
||||
code = "<?php\nclass Broken {\n public function oops( {\n"
|
||||
result = self._compressor().compress(code, language="php")
|
||||
assert result.compressed == code
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue