mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
2ad300aff8
## Problem
pyo3 marks `_native::Parser` as `#[pyclass(unsendable)]`, which causes a
hard thread-assertion panic when a parser created on one thread is
accessed from another:
```
thread '<unnamed>' panicked at pyo3-0.28.3/src/impl_/pyclass.rs:1055:9:
assertion `left == right` failed: _native::Parser is unsendable, but sent to another thread
left: ThreadId(2)
right: ThreadId(1)
```
The prior implementation stored parsers in a module-level `dict[str,
Any]` (`_tree_sitter_languages`). When `_run_compression_in_executor`
dispatched compression work to a `ThreadPoolExecutor`, pool workers
grabbed parsers from that shared dict that were originally created on
the main asyncio thread and panicked.
This produces a 500 on every request where code compression is attempted
via a pool thread.
## Fix
Replace the global dict with `threading.local()` so each thread creates
and owns its own parser instances. No cross-thread parser access is
possible.
```python
# before
_tree_sitter_languages: dict[str, Any] = {} # shared — crosses threads
# after
_tree_sitter_local = threading.local() # per-thread — isolated
```
`is_tree_sitter_loaded()` and `unload_tree_sitter()` updated to operate
on the current thread's local cache (semantics unchanged for
single-threaded callers).
## Tests
9 regression tests added in
`tests/test_transforms/test_tree_sitter_thread_safety.py`:
- Thread isolation: two threads get distinct parser instances
- Within-thread reuse: same thread gets the same cached instance
- Thread pool: parsers usable from `ThreadPoolExecutor` workers without
panic
- Concurrent workers: each distinct pool thread owns a unique parser
- `is_tree_sitter_loaded` / `unload_tree_sitter` lifecycle
Also adds a `filterwarnings` entry for
`PytestUnraisableExceptionWarning`: pyo3 emits this when short-lived
test threads drop parsers at teardown; it does not occur in production
where pool threads are long-lived.
## Relation to #564
PR #564 proposes the same `threading.local()` approach but was blocked
on missing tests (`CHANGES_REQUESTED`). This PR includes the full test
suite.
83 lines
2.4 KiB
Bash
83 lines
2.4 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
profile="${1:-default}"
|
|
project_env="${UV_PROJECT_ENVIRONMENT:-/home/vscode/.venvs/headroom}"
|
|
project_env_root="$(dirname "$project_env")"
|
|
cache_root="${HOME}/.cache"
|
|
script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
workspace_root="$(cd "$script_dir/.." && pwd)"
|
|
git_profile_script="/etc/profile.d/headroom-worktree-git.sh"
|
|
sync_extras=(--extra dev)
|
|
|
|
if [[ "$profile" == "memory-stack" ]]; then
|
|
sync_extras+=(--extra memory-stack)
|
|
fi
|
|
|
|
cd "$workspace_root"
|
|
|
|
configure_worktree_git_env() {
|
|
sudo rm -f "$git_profile_script"
|
|
|
|
if git rev-parse --show-toplevel >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
|
|
if [[ ! -f .git ]]; then
|
|
return 1
|
|
fi
|
|
|
|
local gitdir_spec repo_root translated_git_dir translated_common_dir
|
|
gitdir_spec="$(sed -n 's/^gitdir: //p' .git)"
|
|
|
|
if [[ -z "$gitdir_spec" || "$gitdir_spec" != *"/.git/worktrees/"* ]]; then
|
|
return 1
|
|
fi
|
|
|
|
repo_root="${gitdir_spec%/.git/worktrees/*}"
|
|
|
|
if [[ "$gitdir_spec" =~ ^[A-Za-z]:/ ]]; then
|
|
translated_git_dir="/workspaces-host/${gitdir_spec#?:/}"
|
|
translated_common_dir="/workspaces-host/${repo_root#?:/}/.git"
|
|
elif [[ "$gitdir_spec" == /* ]]; then
|
|
translated_git_dir="/workspaces-host${gitdir_spec}"
|
|
translated_common_dir="/workspaces-host${repo_root}/.git"
|
|
else
|
|
return 1
|
|
fi
|
|
|
|
if [[ ! -d "$translated_git_dir" || ! -d "$translated_common_dir" ]]; then
|
|
return 1
|
|
fi
|
|
|
|
sudo tee "$git_profile_script" >/dev/null <<EOF
|
|
export GIT_DIR="$translated_git_dir"
|
|
export GIT_COMMON_DIR="$translated_common_dir"
|
|
export GIT_WORK_TREE="$workspace_root"
|
|
EOF
|
|
|
|
# shellcheck disable=SC1091
|
|
source "$git_profile_script"
|
|
git rev-parse --show-toplevel >/dev/null 2>&1
|
|
}
|
|
|
|
sudo mkdir -p "$project_env_root" "$cache_root/uv" "$cache_root/pip" "$cache_root/pre-commit"
|
|
sudo chown -R "$(id -u):$(id -g)" "$project_env_root" "$cache_root"
|
|
|
|
uv sync --frozen "${sync_extras[@]}" --link-mode copy
|
|
|
|
if configure_worktree_git_env; then
|
|
uv run pre-commit install
|
|
else
|
|
echo "Skipping pre-commit install because git metadata is not available inside this container."
|
|
fi
|
|
|
|
echo "Headroom devcontainer is ready."
|
|
if [[ "$profile" == "memory-stack" ]]; then
|
|
echo "Memory stack sidecars are available at qdrant:6333 and neo4j://neo4j:7687."
|
|
fi
|
|
echo "Run checks with:"
|
|
echo " uv run ruff check ."
|
|
echo " uv run ruff format --check ."
|
|
echo " uv run mypy headroom --ignore-missing-imports"
|
|
echo " uv run pytest -v --tb=short"
|