mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
efd2ac1ca4
`.gitattributes` declares `*.py text eol=lf` and `*.sh text eol=lf`, but 74 files (73 .py, 1 .sh) are stored in the index with CRLF line endings, violating that contract. Every macOS/Linux clone reports these files as "modified" on fresh checkout because git's diff engine sees the stored bytes don't match the attribute contract, even though the working tree and index match byte-for-byte. Running `git add --renormalize .` rewrites each affected blob so the stored form matches the attribute declaration. No semantic changes — every affected file's diff is "N insertions, N deletions" with inserts and deletes being the same lines modulo line endings. Follow-up commit adds `.git-blame-ignore-revs` so `git blame` / GitHub blame skip this mechanical commit. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
50 lines
1.7 KiB
Python
50 lines
1.7 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from headroom.tokenizer import Tokenizer, count_tokens_messages, count_tokens_text
|
|
|
|
|
|
class FakeTokenCounter:
|
|
def __init__(self) -> None:
|
|
self.calls: list[tuple[str, Any]] = []
|
|
|
|
def count_text(self, text: str) -> int:
|
|
self.calls.append(("text", text))
|
|
return len(text.split())
|
|
|
|
def count_message(self, message: dict[str, Any]) -> int:
|
|
self.calls.append(("message", message))
|
|
return len(str(message.get("content", "")).split())
|
|
|
|
def count_messages(self, messages: list[dict[str, Any]]) -> int:
|
|
self.calls.append(("messages", messages))
|
|
return sum(len(str(msg.get("content", "")).split()) for msg in messages)
|
|
|
|
|
|
def test_tokenizer_delegates_to_counter() -> None:
|
|
counter = FakeTokenCounter()
|
|
tokenizer = Tokenizer(counter, model="gpt-4o")
|
|
|
|
assert tokenizer.model == "gpt-4o"
|
|
assert tokenizer.available is True
|
|
assert tokenizer.count_text("hello world") == 2
|
|
assert tokenizer.count_message({"role": "user", "content": "three word text"}) == 3
|
|
assert tokenizer.count_messages([{"content": "one two"}, {"content": "three"}]) == 3
|
|
assert counter.calls == [
|
|
("text", "hello world"),
|
|
("message", {"role": "user", "content": "three word text"}),
|
|
("messages", [{"content": "one two"}, {"content": "three"}]),
|
|
]
|
|
|
|
|
|
def test_tokenizer_convenience_functions() -> None:
|
|
counter = FakeTokenCounter()
|
|
messages = [{"content": "one"}, {"content": "two three"}]
|
|
|
|
assert count_tokens_text("alpha beta gamma", counter) == 3
|
|
assert count_tokens_messages(messages, counter) == 3
|
|
assert counter.calls == [
|
|
("text", "alpha beta gamma"),
|
|
("messages", messages),
|
|
]
|