mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
## Description `headroom install status` crashed with `OSError: [WinError 87] The parameter is incorrect` on Windows and, worse, tore down the live proxy it was only meant to inspect. `runtime_status()` probed liveness with a bare `os.kill(pid, 0)` guarded only by `except OSError`. Against a detached Windows agent (`DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP`), that call raises WinError 87, which CPython surfaces as a `SystemError` — not an `OSError` — so it escaped the handler, crashed status, and left the deployment dead (PID file removed, port 8787 freed). This mirrors the `os.kill`/`SystemError` fix PR #1315 applied to `cli/wrap.py`. Closes #1544 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Added a shared Windows-safe `headroom._subprocess.pid_alive()` helper: rejects non-positive PIDs, prefers `psutil.pid_exists()`, and treats `SystemError` (WinError 87) as "not alive". - `install/runtime.py` `runtime_status()` now delegates to `pid_alive()` instead of an unguarded `os.kill(pid, 0)`. - `install/runtime.py` `stop_runtime()` now also catches `SystemError` to avoid the same crash class on shutdown. - `cli/wrap.py` `_pid_alive()` now delegates to the shared helper, so the marker-cleanup path and the install/runtime status path share one liveness probe (the shared helper the issue asked for). - Added regression tests for the helper and `runtime_status`. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [x] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [ ] Manual testing performed ### Test Output ```text $ ruff check . All checks passed! $ ruff format --check headroom/_subprocess.py headroom/install/runtime.py headroom/cli/wrap.py tests/test_install/test_runtime.py tests/test_pid_alive.py 5 files already formatted $ mypy headroom --ignore-missing-imports (exit 0) $ pytest tests/test_pid_alive.py tests/test_install/test_runtime.py tests/test_cli/test_wrap_helpers.py tests/test_cli/test_wrap_persistent.py \ --deselect "tests/test_install/test_runtime.py::test_runtime_start_lock_blocks_another_process" 89 passed ``` ## Real Behavior Proof - Environment: Windows 11, Python 3.13, ruff 0.15.17 / mypy 1.20.2 / pytest 9.1.0, psutil 7.2.2, branch `fix/1544-windows-pid-liveness`. - Exact command / steps: ran the four checks above; the new `tests/test_pid_alive.py` injects a `SystemError` (simulated WinError 87) and a stubbed `psutil` to drive both code paths, and `test_runtime_status_*` exercise `runtime_status()` end to end with a PID file present. - Observed result: `runtime_status` returns `"running"` for a live PID without sending any signal (asserted), returns `"stopped"` instead of crashing when the probe raises `SystemError`, and the helper only ever passes signal `0`. All 89 targeted tests pass; ruff/format/mypy clean. - Not tested: the full `headroom install apply --preset persistent-task` detached-agent reproduction against a live proxy was not run end to end; it is instead covered by the deterministic `SystemError`/WinError-87 injection regression tests. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] I have updated the CHANGELOG.md if applicable ## Additional Notes - One pre-existing test, `test_runtime_start_lock_blocks_another_process`, fails on my local Windows checkout **before** these changes too (it asserts cross-process file-lock blocking and depends on `HOME` semantics that differ on Windows). It is unrelated to this fix and is deselected above; it passes on the Linux CI runners. Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
b84afbfb83
commit
6b227b9c90
5 changed files with 149 additions and 32 deletions
|
|
@ -1,7 +1,37 @@
|
|||
import os
|
||||
import subprocess as _sp
|
||||
from typing import Any
|
||||
|
||||
|
||||
def pid_alive(pid: int) -> bool:
|
||||
"""Return True if ``pid`` names a live process (Windows-safe).
|
||||
|
||||
``os.kill(pid, 0)`` is the usual Unix liveness probe, but on Windows it can
|
||||
fail against a detached/stale/invalid PID with ``WinError 87`` ("The
|
||||
parameter is incorrect"). CPython sometimes surfaces that as a
|
||||
``SystemError`` rather than an ``OSError``; since ``SystemError`` is not an
|
||||
``OSError`` subclass, a bare ``except OSError`` lets it escape and crash the
|
||||
caller — and in the detached-agent path it took down the supervised proxy
|
||||
(issue #1544). Prefer ``psutil.pid_exists`` and treat ``SystemError`` as
|
||||
"not alive".
|
||||
"""
|
||||
if pid <= 0:
|
||||
return False # non-positive PIDs are never valid liveness targets
|
||||
try:
|
||||
import psutil # type: ignore[import-untyped] # optional dep, already used elsewhere
|
||||
|
||||
return bool(psutil.pid_exists(pid))
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except PermissionError:
|
||||
return True # exists but owned by another user
|
||||
except (ProcessLookupError, OSError, SystemError):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def run(*args: Any, **kwargs: Any) -> _sp.CompletedProcess:
|
||||
if kwargs.get("text") or kwargs.get("universal_newlines"):
|
||||
kwargs.setdefault("encoding", "utf-8")
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ from collections.abc import Callable
|
|||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
from headroom._subprocess import run
|
||||
from headroom._subprocess import pid_alive, run
|
||||
|
||||
# Fix Windows cp1252 encoding — box-drawing characters require UTF-8
|
||||
if sys.platform == "win32" and hasattr(sys.stdout, "buffer"):
|
||||
|
|
@ -3057,27 +3057,12 @@ def _unregister_proxy_client(port: int) -> None:
|
|||
|
||||
|
||||
def _pid_alive(pid: int) -> bool:
|
||||
"""Return True if ``pid`` names a live process."""
|
||||
if pid <= 0:
|
||||
return False # non-positive PIDs are never valid client markers
|
||||
try:
|
||||
import psutil # type: ignore[import-untyped] # optional dep, already used elsewhere
|
||||
"""Return True if ``pid`` names a live process.
|
||||
|
||||
return bool(psutil.pid_exists(pid))
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except PermissionError:
|
||||
return True # exists but owned by another user
|
||||
except (ProcessLookupError, OSError, SystemError):
|
||||
# On Windows, os.kill against a stale/invalid PID can fail with WinError
|
||||
# 87 ("The parameter is incorrect"); CPython sometimes surfaces this as a
|
||||
# SystemError rather than an OSError. SystemError is not an OSError
|
||||
# subclass, so a bare `except OSError` lets it escape and crash cleanup(),
|
||||
# leaving the shared proxy running.
|
||||
return False
|
||||
return True
|
||||
Thin wrapper over the shared Windows-safe helper so the marker-cleanup path
|
||||
and the install/runtime status path use one liveness probe (see #1544).
|
||||
"""
|
||||
return pid_alive(pid)
|
||||
|
||||
|
||||
def _marker_pid_reused(marker: Path, pid: int) -> bool:
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from contextlib import contextmanager
|
|||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
from headroom._subprocess import run
|
||||
from headroom._subprocess import pid_alive, run
|
||||
|
||||
from .health import probe_ready
|
||||
from .models import DeploymentManifest, InstallPreset, RuntimeKind
|
||||
|
|
@ -321,7 +321,8 @@ def stop_runtime(manifest: DeploymentManifest) -> None:
|
|||
return
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except OSError:
|
||||
except (OSError, SystemError):
|
||||
# SystemError covers the Windows WinError 87 surfacing described in #1544.
|
||||
pass
|
||||
_clear_pid(manifest.profile)
|
||||
|
||||
|
|
@ -351,8 +352,7 @@ def runtime_status(manifest: DeploymentManifest) -> str:
|
|||
pid = _read_pid(manifest.profile)
|
||||
if pid is None:
|
||||
return "stopped"
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except OSError:
|
||||
return "stopped"
|
||||
return "running"
|
||||
# Windows-safe liveness probe: a bare os.kill(pid, 0) here raised WinError 87
|
||||
# as a SystemError against the detached agent, crashing status and taking the
|
||||
# live proxy down with it (#1544).
|
||||
return "running" if pid_alive(pid) else "stopped"
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import os
|
|||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
from headroom.install.models import DeploymentManifest, InstallPreset
|
||||
|
|
@ -466,9 +467,7 @@ def test_start_stop_wait_and_runtime_status_branches(monkeypatch, tmp_path: Path
|
|||
assert runtime_status(python_manifest) == "stopped"
|
||||
|
||||
_write_pid("default", 125)
|
||||
monkeypatch.setattr(
|
||||
"headroom.install.runtime.os.kill", lambda pid, sig: (_ for _ in ()).throw(OSError())
|
||||
)
|
||||
monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: False)
|
||||
assert runtime_status(python_manifest) == "stopped"
|
||||
|
||||
|
||||
|
|
@ -530,7 +529,7 @@ def test_runtime_status_reads_container_and_pid_state(monkeypatch, tmp_path: Pat
|
|||
pid_file = tmp_path / ".headroom" / "deploy" / "default" / "runner.pid"
|
||||
pid_file.parent.mkdir(parents=True)
|
||||
pid_file.write_text("123", encoding="utf-8")
|
||||
monkeypatch.setattr("headroom.install.runtime.os.kill", lambda pid, sig: None)
|
||||
monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)
|
||||
python_manifest = DeploymentManifest(
|
||||
profile="default",
|
||||
preset="persistent-service",
|
||||
|
|
@ -544,3 +543,56 @@ def test_runtime_status_reads_container_and_pid_state(monkeypatch, tmp_path: Pat
|
|||
backend="anthropic",
|
||||
)
|
||||
assert runtime_status(python_manifest) == "running"
|
||||
|
||||
|
||||
def _python_service_manifest() -> DeploymentManifest:
|
||||
return DeploymentManifest(
|
||||
profile="default",
|
||||
preset="persistent-service",
|
||||
runtime_kind="python",
|
||||
supervisor_kind="service",
|
||||
scope="user",
|
||||
provider_mode="manual",
|
||||
targets=[],
|
||||
port=8787,
|
||||
host="127.0.0.1",
|
||||
backend="anthropic",
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_status_reports_live_pid_without_terminating(monkeypatch, tmp_path: Path) -> None:
|
||||
"""#1544: status on a live detached PID stays 'running' and never signals it."""
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
pid_file = tmp_path / ".headroom" / "deploy" / "default" / "runner.pid"
|
||||
pid_file.parent.mkdir(parents=True)
|
||||
pid_file.write_text("25212", encoding="utf-8")
|
||||
|
||||
def fail_kill(pid: int, sig: int) -> None:
|
||||
raise AssertionError(f"status must not signal the live proxy (pid={pid}, sig={sig})")
|
||||
|
||||
monkeypatch.setattr("headroom.install.runtime.os.kill", fail_kill)
|
||||
monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)
|
||||
|
||||
assert runtime_status(_python_service_manifest()) == "running"
|
||||
assert pid_file.exists() # status left the deployment untouched
|
||||
|
||||
|
||||
def test_runtime_status_survives_winerror87_systemerror(monkeypatch, tmp_path: Path) -> None:
|
||||
"""#1544: a WinError 87 SystemError from the liveness probe yields 'stopped', not a crash."""
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
pid_file = tmp_path / ".headroom" / "deploy" / "default" / "runner.pid"
|
||||
pid_file.parent.mkdir(parents=True)
|
||||
pid_file.write_text("25212", encoding="utf-8")
|
||||
|
||||
# Force the psutil fast-path to bail so the os.kill fallback runs...
|
||||
fake_psutil = types.SimpleNamespace(
|
||||
pid_exists=lambda pid: (_ for _ in ()).throw(RuntimeError("no psutil"))
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "psutil", fake_psutil)
|
||||
# ...where Windows surfaces WinError 87 as a SystemError, not an OSError.
|
||||
monkeypatch.setattr(
|
||||
"headroom._subprocess.os.kill",
|
||||
lambda pid, sig: (_ for _ in ()).throw(SystemError("WinError 87")),
|
||||
)
|
||||
|
||||
assert runtime_status(_python_service_manifest()) == "stopped"
|
||||
|
|
|
|||
50
tests/test_pid_alive.py
Normal file
50
tests/test_pid_alive.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
"""Regression tests for the Windows-safe PID liveness helper (#1544)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import types
|
||||
|
||||
from headroom._subprocess import pid_alive
|
||||
|
||||
|
||||
def test_pid_alive_rejects_non_positive() -> None:
|
||||
assert pid_alive(0) is False
|
||||
assert pid_alive(-1) is False
|
||||
|
||||
|
||||
def test_pid_alive_prefers_psutil_without_signalling(monkeypatch) -> None:
|
||||
monkeypatch.setitem(sys.modules, "psutil", types.SimpleNamespace(pid_exists=lambda pid: True))
|
||||
|
||||
def boom(pid: int, sig: int) -> None:
|
||||
raise AssertionError("os.kill must not run when psutil answers")
|
||||
|
||||
monkeypatch.setattr("headroom._subprocess.os.kill", boom)
|
||||
assert pid_alive(4321) is True
|
||||
|
||||
|
||||
def test_pid_alive_systemerror_is_not_alive(monkeypatch) -> None:
|
||||
"""WinError 87 surfaces as SystemError on Windows; it must read as 'not alive', not crash."""
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"psutil",
|
||||
types.SimpleNamespace(pid_exists=lambda pid: (_ for _ in ()).throw(RuntimeError())),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"headroom._subprocess.os.kill",
|
||||
lambda pid, sig: (_ for _ in ()).throw(SystemError("WinError 87")),
|
||||
)
|
||||
assert pid_alive(4321) is False
|
||||
|
||||
|
||||
def test_pid_alive_only_uses_signal_zero(monkeypatch) -> None:
|
||||
"""The liveness probe must never send a real (terminating) signal."""
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"psutil",
|
||||
types.SimpleNamespace(pid_exists=lambda pid: (_ for _ in ()).throw(RuntimeError())),
|
||||
)
|
||||
sent: list[int] = []
|
||||
monkeypatch.setattr("headroom._subprocess.os.kill", lambda pid, sig: sent.append(sig))
|
||||
assert pid_alive(4321) is True
|
||||
assert sent == [0]
|
||||
Loading…
Add table
Add a link
Reference in a new issue