fix(health): readyz verifies upstream connectivity, not just process liveness (#744)

Closes #740

## What

`/readyz` and `/health` previously reported healthy even when the
upstream API was completely unreachable (e.g. SSL certificate errors,
wrong URL, network failure). The proxy would accept traffic and return
502 on every `/v1/messages` request.

## Changes

- Added `_check_upstream()` async function that probes the configured
upstream base URL with a HEAD request (5s timeout, result cached 30s) to
verify TLS + TCP reachability without triggering an inference call
- `/readyz` now calls `_check_upstream()` before building its response;
returns HTTP 503 if the upstream is unreachable
- `/health` exposes an `upstream` sub-check entry with `enabled`,
`ready`, `status`, and `error` fields
- `HEADROOM_SKIP_UPSTREAM_CHECK=1` opts out (for air-gapped or test
environments)
- Existing tests updated to set `HEADROOM_SKIP_UPSTREAM_CHECK=1` so unit
tests don't make live network calls
- Three new tests covering: opt-out via env var, 503 on upstream
failure, `/health` includes upstream check

## Behaviour

| Endpoint | Before | After |
|---|---|---|
| `/livez` | process alive | unchanged |
| `/readyz` | process alive | process alive AND upstream reachable |
| `/health` | no upstream info | includes `checks.upstream` with status
+ error |

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

headroom/proxy/server.py

@@ -1601,6 +1601,12 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
                 native_tool=bool(memory_status.get("native_tool", False)),
                 bridge_enabled=bool(memory_status.get("bridge_enabled", False)),
             ),
+            "upstream": _component_health(
+                enabled=os.environ.get("HEADROOM_SKIP_UPSTREAM_CHECK", "").strip() != "1",
+                ready=bool(_upstream_check_cache["ok"]),
+                url=_upstream_check_cache["url"],
+                error=_upstream_check_cache["error"],
+            ),
         }
 
     def _runtime_payload() -> dict[str, Any]:
@@ -1703,6 +1709,70 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
             }
         return payload
 
+    # ---------------------------------------------------------------------------
+    # Upstream connectivity check — cached to avoid hammering the upstream on
+    # every /readyz poll.  Set HEADROOM_SKIP_UPSTREAM_CHECK=1 to opt out (e.g.
+    # in air-gapped or test environments where the upstream isn't reachable at
+    # startup time).
+    # ---------------------------------------------------------------------------
+
+    _UPSTREAM_CHECK_TTL = 30.0  # seconds
+    _upstream_check_cache: dict[str, Any] = {
+        "expires_at": 0.0,
+        "ok": True,
+        "error": None,
+        "url": None,
+    }
+    _upstream_check_lock = asyncio.Lock()
+
+    def _upstream_target_url() -> str:
+        """Return the primary upstream base URL to probe."""
+        # Use the resolved API target from the provider runtime so we respect
+        # any overrides set by ProxyConfig.anthropic_api_url / env vars.
+        return proxy.provider_runtime.api_targets.anthropic
+
+    async def _check_upstream() -> None:
+        """Probe the upstream API endpoint and update the cached result.
+
+        Uses a HEAD request with a 5-second timeout — just enough to verify
+        TLS + TCP reachability without triggering an inference call.
+        """
+        if os.environ.get("HEADROOM_SKIP_UPSTREAM_CHECK", "").strip() == "1":
+            # Opt-out: treat upstream as always reachable.
+            _upstream_check_cache["ok"] = True
+            _upstream_check_cache["error"] = None
+            _upstream_check_cache["expires_at"] = time.monotonic() + _UPSTREAM_CHECK_TTL
+            return
+
+        now = time.monotonic()
+        # Fast-path: return if the cached result is still fresh (no lock needed
+        # for a simple float comparison — worst case we re-check twice).
+        if now < _upstream_check_cache["expires_at"]:
+            return
+
+        async with _upstream_check_lock:
+            # Re-check inside the lock to handle concurrent waiters.
+            if time.monotonic() < _upstream_check_cache["expires_at"]:
+                return
+            url = _upstream_target_url()
+            _upstream_check_cache["url"] = url
+            client = proxy.http_client
+            if client is None:
+                _upstream_check_cache["ok"] = False
+                _upstream_check_cache["error"] = "proxy client not initialised"
+                _upstream_check_cache["expires_at"] = time.monotonic() + _UPSTREAM_CHECK_TTL
+                return
+            try:
+                resp = await client.head(url, timeout=5.0)
+                # Any HTTP response (even 4xx/5xx) means TLS+TCP worked.
+                _ = resp.status_code
+                _upstream_check_cache["ok"] = True
+                _upstream_check_cache["error"] = None
+            except Exception as exc:  # noqa: BLE001
+                _upstream_check_cache["ok"] = False
+                _upstream_check_cache["error"] = str(exc)
+            _upstream_check_cache["expires_at"] = time.monotonic() + _UPSTREAM_CHECK_TTL
+
     # CORS
     app.add_middleware(
         CORSMiddleware,
@@ -1838,11 +1908,13 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
 
     @app.get("/readyz")
     async def readyz():
+        await _check_upstream()
         payload = _health_payload(include_config=False)
         return JSONResponse(status_code=200 if payload["ready"] else 503, content=payload)
 
     @app.get("/health")
     async def health():
+        await _check_upstream()
         payload = _health_payload(include_config=True)
         return JSONResponse(status_code=200, content=payload)
 

tests/test_proxy_healthchecks.py

@@ -12,7 +12,10 @@ from headroom.proxy.server import ProxyConfig, create_app
 
 
 @pytest.fixture
-def client():
+def client(monkeypatch):
+    # Skip the live upstream connectivity probe in unit tests — tests verify
+    # the check logic separately (see test_readyz_upstream_check_* below).
+    monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
     config = ProxyConfig(
         optimize=False,
         cache_enabled=False,
@@ -80,6 +83,7 @@ def test_health_preserves_backwards_compatible_config_payload(client):
 
 
 def test_health_includes_deployment_metadata_when_present(monkeypatch):
+    monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
     monkeypatch.setenv("HEADROOM_DEPLOYMENT_PROFILE", "default")
     monkeypatch.setenv("HEADROOM_DEPLOYMENT_PRESET", "persistent-service")
     monkeypatch.setenv("HEADROOM_DEPLOYMENT_RUNTIME", "python")
@@ -116,7 +120,8 @@ def test_health_remains_200_when_proxy_is_not_ready(client):
     assert response.json()["ready"] is False
 
 
-def test_readyz_reports_memory_backend_when_enabled(tmp_path):
+def test_readyz_reports_memory_backend_when_enabled(tmp_path, monkeypatch):
+    monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
     config = ProxyConfig(
         optimize=False,
         cache_enabled=False,
@@ -141,6 +146,7 @@ def test_readyz_reports_memory_backend_when_enabled(tmp_path):
 
 
 def test_readyz_initializes_qdrant_memory_backend(monkeypatch):
+    monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
     from headroom.memory.backends import direct_mem0
 
     init_calls: list[str] = []
@@ -175,7 +181,8 @@ def test_readyz_initializes_qdrant_memory_backend(monkeypatch):
     assert data["checks"]["memory"]["initialized"] is True
 
 
-def test_shutdown_tolerates_stubbed_memory_handler():
+def test_shutdown_tolerates_stubbed_memory_handler(monkeypatch):
+    monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
     config = ProxyConfig(
         optimize=False,
         cache_enabled=False,
@@ -197,3 +204,86 @@ def test_shutdown_tolerates_stubbed_memory_handler():
         response = client.get("/health")
 
     assert response.status_code == 200
+
+
+# ---------------------------------------------------------------------------
+# Upstream connectivity check tests
+# ---------------------------------------------------------------------------
+
+
+def test_readyz_upstream_check_disabled_by_env_var(monkeypatch):
+    """HEADROOM_SKIP_UPSTREAM_CHECK=1 suppresses the probe and reports ready."""
+    monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
+    config = ProxyConfig(
+        optimize=False,
+        cache_enabled=False,
+        rate_limit_enabled=False,
+        cost_tracking_enabled=False,
+    )
+    app = create_app(config)
+
+    with TestClient(app) as test_client:
+        response = test_client.get("/readyz")
+
+    assert response.status_code == 200
+    data = response.json()
+    assert data["ready"] is True
+    # When the check is skipped the component is reported as "disabled"
+    assert data["checks"]["upstream"]["enabled"] is False
+    assert data["checks"]["upstream"]["ready"] is True
+
+
+def test_readyz_upstream_check_failure_returns_503(monkeypatch):
+    """A failed upstream probe makes /readyz return HTTP 503."""
+    from unittest.mock import AsyncMock, patch
+
+    import httpx
+
+    monkeypatch.delenv("HEADROOM_SKIP_UPSTREAM_CHECK", raising=False)
+
+    config = ProxyConfig(
+        optimize=False,
+        cache_enabled=False,
+        rate_limit_enabled=False,
+        cost_tracking_enabled=False,
+    )
+    app = create_app(config)
+
+    # Patch the proxy's shared http_client.head so the probe uses the same
+    # client as real traffic (which also means TLS/CA config is consistent).
+    with TestClient(app) as test_client:
+        with patch.object(
+            test_client.app.state.proxy.http_client,
+            "head",
+            new=AsyncMock(side_effect=httpx.ConnectError("connection refused (test)")),
+        ):
+            response = test_client.get("/readyz")
+
+    assert response.status_code == 503
+    data = response.json()
+    assert data["ready"] is False
+    assert data["checks"]["upstream"]["ready"] is False
+    assert "connection refused" in data["checks"]["upstream"]["error"]
+
+
+def test_health_includes_upstream_check_result(monkeypatch):
+    """/health always returns 200 but exposes the upstream check result."""
+    monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
+    config = ProxyConfig(
+        optimize=False,
+        cache_enabled=False,
+        rate_limit_enabled=False,
+        cost_tracking_enabled=False,
+    )
+    app = create_app(config)
+
+    with TestClient(app) as test_client:
+        response = test_client.get("/health")
+
+    assert response.status_code == 200
+    data = response.json()
+    assert "upstream" in data["checks"]
+    upstream = data["checks"]["upstream"]
+    assert "enabled" in upstream
+    assert "ready" in upstream
+    assert "status" in upstream