From 4c3e822d92fd8554dc67688d02dbfe216d22b4b7 Mon Sep 17 00:00:00 2001 From: JerrettDavis Date: Thu, 27 Aug 2026 12:56:38 -0500 Subject: [PATCH] test(copilot): add authenticated Windows E2E coverage --- README.md | 8 +- TESTING-copilot-subscription.md | 44 ++-- docs/content/docs/vscode-copilot.mdx | 11 +- e2e/copilot_live.py | 338 +++++++++++++++++++++++++++ headroom/cli/wrap.py | 15 ++ tests/test_cli/test_wrap_helpers.py | 23 ++ 6 files changed, 420 insertions(+), 19 deletions(-) create mode 100644 e2e/copilot_live.py diff --git a/README.md b/README.md index 0b923e940..929dfc8c5 100644 --- a/README.md +++ b/README.md @@ -283,7 +283,13 @@ For GitHub.com Enterprise Cloud URLs such as override. Headroom uses GitHub's normal token-exchange endpoint and the Copilot API endpoint advertised for the signed-in account. -Platform support note: macOS auth reuse via Copilot CLI Keychain storage has been smoke-tested. Windows Credential Manager, Linux Secret Service / `secret-tool`, and Docker/CI token-injection paths are implemented or planned as auth-discovery paths, but still need real OS validation before they should be considered fully vetted. For Docker and CI, prefer passing an explicit `GITHUB_COPILOT_TOKEN` or `GITHUB_COPILOT_GITHUB_TOKEN` rather than relying on host keychain access. +Platform support note: macOS auth reuse via Copilot CLI Keychain storage and +Windows Headroom device authentication have been live-tested. Copilot CLI 1.0.81 +does not expose its working Windows login through the legacy Credential Manager +schema Headroom recognizes, so run `headroom copilot-auth login` on Windows. +Linux Secret Service / `secret-tool` reuse still needs real OS validation. For +Docker and CI, prefer passing an explicit `GITHUB_COPILOT_TOKEN` or +`GITHUB_COPILOT_GITHUB_TOKEN` rather than relying on host keychain access. ### GitHub Copilot in Visual Studio Code diff --git a/TESTING-copilot-subscription.md b/TESTING-copilot-subscription.md index 2255ccb28..e5ab74c84 100644 --- a/TESTING-copilot-subscription.md +++ b/TESTING-copilot-subscription.md @@ -1,12 +1,11 @@ # Testing: GitHub Copilot subscription mode (`headroom wrap copilot --subscription`) -This is an **experimental** feature and we need help verifying it on **Linux and -Windows**. It already works on macOS; the cross-platform gap is small and -specific (see [Status](#status)). If you have a GitHub Copilot subscription and -10 minutes, please run one of the flows below and +This feature has live coverage on macOS and Windows. Additional Linux secret-store +coverage is still useful (see [Status](#status)). If you have a GitHub Copilot +subscription and 10 minutes, please run one of the flows below and [file a report](https://github.com/chopratejas/headroom/issues/new?template=copilot-subscription-test-report.md). -> ⚠️ This is experimental, and it reads your Copilot login token + routes your +> ⚠️ This reads your Copilot login token and routes your > Copilot CLI traffic through a local Headroom proxy. Only run it if you're > comfortable with that. The branch is open for inspection. @@ -61,7 +60,8 @@ real enterprise tenant. |----------|:---:|:---:| | macOS (Keychain) | ✅ verified | ✅ verified (`copilot-cli`) | | Linux (`secret-tool`/libsecret) | ✅ expected | ❓ **needs testing** | -| Windows (Credential Manager) | ✅ expected | ❓ **needs testing** | +| Windows (Headroom device auth) | ✅ verified | ✅ verified | +| Windows (Copilot CLI credential reuse) | ✅ verified after auth | ❌ Copilot CLI 1.0.81 does not expose the legacy Credential Manager schema | | Any OS via `GITHUB_COPILOT_TOKEN` env var | ✅ verified by tests | n/a (bypasses discovery) | The two things we want to learn: @@ -103,7 +103,24 @@ headroom wrap copilot --subscription -- --model gpt-4o -p "Reply with exactly: H ## Windows -There is **no native Windows wheel yet**, so pick one: +For a source checkout with Python and Rust installed, build the current tree with +the proxy extra and authorize Headroom's dedicated OAuth app: + +```powershell +uv sync --extra proxy --extra dev +uv run --no-sync headroom copilot-auth login +uv run --no-sync python e2e/copilot_live.py --vscode-extension ` + --model gpt-5-mini --model gpt-5.5 ` + --model gpt-5.6-luna --model gpt-5.6-sol --model gpt-5.6-terra +``` + +The live suite uses the official Copilot CLI, exercises subscription wrapping, +sends requests through an isolated VS Code proxy configuration, and optionally +drives the installed VS Code extension through `code chat`. It snapshots and +restores real VS Code settings byte-for-byte and never reads or prints token +values. + +Packaged-install alternatives: **A. Mechanism test (easiest — Docker Desktop or WSL2):** ```powershell @@ -114,16 +131,17 @@ headroom wrap copilot --subscription -- --model gpt-4o -p "Reply with: HEADROOM_ ``` Report whether it prints `HEADROOM_OK`. -**B. Native auto-discovery schema (even without a working install):** after -`copilot` login, tell us where Windows stored the token: +**B. Native auto-discovery schema:** after `copilot` login, check whether the +installed Copilot CLI exposes a reusable Windows credential target: ```cmd cmd /c "cmdkey /list" ``` -Report the `Target:` line that looks Copilot-related (it shows the target name, -not the secret). That single fact lets us make native Windows discovery work. +Report only a Copilot-related `Target:` line (it shows the target name, not the +secret). Copilot CLI 1.0.81 did not expose such a target in live Windows testing, +so use `headroom copilot-auth login` when native reuse is unavailable. -> Native Windows auto-discovery becomes fully testable once we add a Windows -> wheel to the build matrix — tracked separately. +> A native Windows wheel is still tracked separately; source builds can run the +> full Windows authentication and routing matrix today. --- diff --git a/docs/content/docs/vscode-copilot.mdx b/docs/content/docs/vscode-copilot.mdx index 003224905..41a8c3299 100644 --- a/docs/content/docs/vscode-copilot.mdx +++ b/docs/content/docs/vscode-copilot.mdx @@ -103,17 +103,18 @@ Chat Completions API. Headroom proxies both routes. Do not treat a model's `unsupported_api_for_model` response from `/chat/completions` as a proxy failure; VS Code uses the endpoint supported by that model. -Live verification on July 31, 2026 confirmed the account catalog and a successful -HTTP 200 response through Headroom's `/responses` route for each of these exact -model IDs: +Live verification on August 27, 2026 confirmed successful responses through +Headroom's `/responses` route for each of these requested model IDs: - `gpt-5.5` - `gpt-5.6-luna` - `gpt-5.6-sol` - `gpt-5.6-terra` -Each response retained the requested model ID. Model availability remains subject -to the signed-in user's Copilot plan and organization policy. +Headroom retained each requested model alias. GitHub may identify the resolved +snapshot in response metadata (for example, a request for `gpt-5.5` returned +`gpt-5.5-2026-04-23`). Model availability remains subject to the signed-in user's +Copilot plan and organization policy. The proxy supports the native Copilot OpenAI-compatible request paths used by GPT and Claude models. Headroom's upstream auth hook replaces local client auth diff --git a/e2e/copilot_live.py b/e2e/copilot_live.py new file mode 100644 index 000000000..ed6400111 --- /dev/null +++ b/e2e/copilot_live.py @@ -0,0 +1,338 @@ +"""Credential-safe live E2E checks for Headroom's GitHub Copilot integrations. + +Run from a source checkout after ``headroom copilot-auth login``:: + + uv run --no-sync python e2e/copilot_live.py + +The script never reads or prints a credential. It uses an isolated VS Code +settings file, verifies restoration byte-for-byte, and sends small live prompts +through both the Copilot CLI wrapper and the VS Code proxy route. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import quote +from urllib.request import Request, urlopen + + +def run(command: list[str], *, timeout: float = 120) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + command, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=timeout, + ) + if result.returncode: + raise RuntimeError( + f"command failed ({result.returncode}): {' '.join(command[:4])}\n" + f"{result.stdout}{result.stderr}" + ) + return result + + +def wait_for_health(port: int, timeout: float = 45) -> dict[str, object]: + deadline = time.monotonic() + timeout + url = f"http://127.0.0.1:{port}/health" + while time.monotonic() < deadline: + try: + with urlopen(url, timeout=2) as response: # noqa: S310 - loopback E2E target + return json.load(response) + except (OSError, URLError, json.JSONDecodeError): + time.sleep(0.25) + raise TimeoutError(f"Headroom did not become healthy on port {port}") + + +def post_response(port: int, project: str, model: str) -> dict[str, object]: + payload = json.dumps( + {"model": model, "input": "Reply with exactly: HEADROOM_VSCODE_OK", "stream": False} + ).encode() + url = f"http://127.0.0.1:{port}/p/{quote(project, safe='')}/v1/responses" + request = Request(url, data=payload, headers={"Content-Type": "application/json"}) + try: + with urlopen(request, timeout=120) as response: # noqa: S310 - loopback E2E target + return json.load(response) + except HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"live VS Code route returned HTTP {exc.code}: {body}") from exc + + +def request_count(port: int) -> float: + with urlopen(f"http://127.0.0.1:{port}/metrics", timeout=5) as response: # noqa: S310 + metrics = response.read().decode("utf-8", errors="replace") + match = re.search(r"^headroom_requests_total ([0-9.eE+-]+)$", metrics, re.MULTILINE) + if not match: + raise AssertionError("Headroom metrics did not expose headroom_requests_total") + return float(match.group(1)) + + +def stop_process(process: subprocess.Popen[str]) -> str: + if process.poll() is None: + if os.name == "nt": + process.send_signal(signal.CTRL_BREAK_EVENT) + else: + process.send_signal(signal.SIGINT) + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.terminate() + process.wait(timeout=10) + stdout, _ = process.communicate() + return stdout + + +def assert_safe_settings(settings: str, proxy_url: str) -> None: + required = ( + "github.copilot.advanced.debug.overrideProxyUrl", + "github.copilot.advanced.debug.overrideCapiUrl", + ) + for key in required: + if key not in settings or proxy_url not in settings: + raise AssertionError(f"VS Code settings did not contain {key}") + forbidden = ("overrideAuthType", "token", "bearer", '"model"') + for value in forbidden: + if value.lower() in settings.lower(): + raise AssertionError(f"VS Code settings unexpectedly contained {value}") + + +def wait_for_settings(path: Path, proxy_url: str, timeout: float = 15) -> str: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + settings = path.read_text(encoding="utf-8") + if all( + key in settings + for key in ( + "github.copilot.advanced.debug.overrideProxyUrl", + "github.copilot.advanced.debug.overrideCapiUrl", + ) + ): + assert_safe_settings(settings, proxy_url) + return settings + time.sleep(0.1) + raise TimeoutError("Headroom did not finish writing the VS Code Copilot settings block") + + +def response_text(payload: dict[str, object]) -> str: + """Return text fragments without depending on one Responses API SDK shape.""" + + fragments: list[str] = [] + + def visit(value: object) -> None: + if isinstance(value, dict): + for key, child in value.items(): + if key in {"text", "output_text"} and isinstance(child, str): + fragments.append(child) + else: + visit(child) + elif isinstance(value, list): + for child in value: + visit(child) + + visit(payload) + return "\n".join(fragments) + + +def model_matches_request(requested: str, returned: object) -> bool: + """Allow GitHub's dated canonical name for an otherwise preserved alias.""" + + return isinstance(returned, str) and ( + returned == requested or returned.startswith(f"{requested}-") + ) + + +def verify_real_vscode_extension( + *, headroom: str, code: str, port: int, settings_path: Path +) -> None: + """Drive the installed VS Code Copilot extension through ``code chat``.""" + + if os.name != "nt": + raise RuntimeError("--vscode-extension currently targets the Windows release gate") + code_binary = shutil.which(code) + if not code_binary: + raise RuntimeError(f"VS Code executable not found on PATH: {code}") + settings_existed = settings_path.exists() + original = settings_path.read_bytes() if settings_existed else None + if original and b"Headroom Copilot proxy" in original: + raise RuntimeError("real VS Code settings already contain a Headroom-managed block") + + process = subprocess.Popen( + [headroom, "wrap", "vscode", "--port", str(port)], + cwd=Path.cwd(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, + ) + output = "" + try: + deadline = time.monotonic() + 45 + proxy_url = "" + while time.monotonic() < deadline: + if settings_path.exists(): + settings = settings_path.read_text(encoding="utf-8") + match = re.search(r'overrideCapiUrl"\s*:\s*"(http://127\.0\.0\.1:\d+)', settings) + if match: + proxy_url = match.group(1) + break + if process.poll() is not None: + raise RuntimeError("VS Code wrapper exited before configuring settings") + time.sleep(0.1) + if not proxy_url: + raise TimeoutError("VS Code wrapper did not configure the real user settings") + actual_port = int(proxy_url.rsplit(":", 1)[1]) + wait_for_health(actual_port) + before = request_count(actual_port) + run( + [ + code_binary, + "chat", + "-m", + "ask", + "-r", + "Reply with exactly: HEADROOM_VSCODE_EXTENSION_OK", + ], + timeout=30, + ) + deadline = time.monotonic() + 90 + while time.monotonic() < deadline and request_count(actual_port) <= before: + time.sleep(1) + if request_count(actual_port) <= before: + raise TimeoutError("the installed VS Code extension sent no request through Headroom") + finally: + output = stop_process(process) + run([headroom, "unwrap", "vscode"]) + + if settings_existed: + if settings_path.read_bytes() != original: + raise AssertionError("real VS Code settings were not restored byte-for-byte") + elif settings_path.exists() and settings_path.read_bytes().strip() not in {b"", b"{}"}: + raise AssertionError("VS Code settings were created but not restored to an empty state") + if "remained running after shutdown" in output.lower(): + raise AssertionError("Windows wrapper orphaned its dedicated proxy") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--headroom", default="headroom", help="Headroom executable to test") + parser.add_argument("--copilot", default="copilot", help="Copilot CLI executable to test") + parser.add_argument("--model", action="append", default=[], help="Live model ID (repeatable)") + parser.add_argument("--port", type=int, default=28787) + parser.add_argument( + "--vscode-extension", + action="store_true", + help="also modify real VS Code settings and drive the installed extension via `code chat`", + ) + parser.add_argument("--code", default="code", help="VS Code executable to test") + args = parser.parse_args() + models = args.model or ["gpt-5-mini"] + + status = run([args.headroom, "copilot-auth", "status"]) + if "Status: logged in" not in status.stdout: + raise RuntimeError("Headroom Copilot auth is missing; run `headroom copilot-auth login`") + + baseline = run( + [args.copilot, "-p", "Reply with exactly: COPILOT_BASELINE_OK", "--model", models[0]] + ) + if "COPILOT_BASELINE_OK" not in baseline.stdout: + raise AssertionError("plain Copilot CLI did not return its sentinel") + + wrapped = run( + [ + args.headroom, + "wrap", + "copilot", + "--subscription", + "--", + "--model", + models[0], + "-p", + "Reply with exactly: HEADROOM_CLI_OK", + ] + ) + if "HEADROOM_CLI_OK" not in wrapped.stdout: + raise AssertionError("wrapped Copilot CLI did not return its sentinel") + + original = b'{\n // preserved by Headroom\n "editor.fontSize": 15,\n}\n' + with tempfile.TemporaryDirectory(prefix="headroom-copilot-e2e-") as temp: + settings_path = Path(temp) / "settings.json" + settings_path.write_bytes(original) + command = [ + args.headroom, + "wrap", + "vscode", + "--port", + str(args.port), + "--settings-file", + str(settings_path), + ] + creationflags = subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0 + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + creationflags=creationflags, + ) + output = "" + try: + health = wait_for_health(args.port) + upstream = str(health.get("config", {}).get("openai_api_url", "")) + if "githubcopilot.com" not in upstream: + raise AssertionError(f"unexpected Copilot upstream host: {upstream}") + project = Path.cwd().name + proxy_url = f"http://127.0.0.1:{args.port}/p/{quote(project, safe='')}" + wait_for_settings(settings_path, proxy_url) + for model in models: + response = post_response(args.port, project, model) + returned_model = response.get("model") + if not model_matches_request(model, returned_model): + raise AssertionError( + f"model was not preserved: requested {model!r}, got {returned_model!r}" + ) + if "HEADROOM_VSCODE_OK" not in response_text(response): + raise AssertionError(f"model {model!r} did not return the expected sentinel") + finally: + output = stop_process(process) + run([args.headroom, "unwrap", "vscode", "--settings-file", str(settings_path)]) + if settings_path.read_bytes() != original: + raise AssertionError("VS Code settings were not restored byte-for-byte") + if any(marker in output.lower() for marker in ("authorization: bearer", "github token")): + raise AssertionError("wrapper output contained a credential-shaped marker") + + if args.vscode_extension: + appdata = os.environ.get("APPDATA", "") + if not appdata: + raise RuntimeError("APPDATA is required to locate stable VS Code settings on Windows") + verify_real_vscode_extension( + headroom=args.headroom, + code=args.code, + port=args.port + 100, + settings_path=Path(appdata) / "Code" / "User" / "settings.json", + ) + + print( + "PASS: Copilot baseline, CLI wrap, VS Code routing, model preservation, restore" + + (", and installed VS Code extension" if args.vscode_extension else "") + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/headroom/cli/wrap.py b/headroom/cli/wrap.py index 966d150ac..2f38e4898 100644 --- a/headroom/cli/wrap.py +++ b/headroom/cli/wrap.py @@ -4601,6 +4601,19 @@ def _make_cleanup(proxy_proc_holder: list, port: int | list[int] = 8787) -> Any: if _other_clients_exist(): # Other clients still using the proxy — leave it running. return + # Snapshot the serving PID before terminating the launcher. On + # Windows the detached serving child can briefly make /health + # unavailable while the launcher exits, causing the later safety + # probe to classify our own listener as "unidentified" and leave + # it orphaned. We still verify it through Headroom's health + # payload before trusting the PID. + serving_pid: int | None = None + if sys.platform == "win32" and _check_proxy(p): + running_config = _query_proxy_config(p) + try: + serving_pid = int(running_config["pid"]) if running_config else None + except (KeyError, TypeError, ValueError): + serving_pid = None if proc.poll() is None: proc.terminate() try: @@ -4614,6 +4627,8 @@ def _make_cleanup(proxy_proc_holder: list, port: int | list[int] = 8787) -> Any: # Ctrl+C from the last wrapper must still stop the listener. if sys.platform == "win32" and _check_proxy(p): stop_status = _stop_local_proxy_for_unwrap(p) + if stop_status == "unidentified" and serving_pid is not None: + stop_status = "stopped" if _kill_proxy_by_pid(serving_pid, p) else "failed" if stop_status not in {"stopped", "not_running"}: click.echo( f" Warning: proxy on port {p} remained running " diff --git a/tests/test_cli/test_wrap_helpers.py b/tests/test_cli/test_wrap_helpers.py index f53df49ba..4cb33b7fa 100644 --- a/tests/test_cli/test_wrap_helpers.py +++ b/tests/test_cli/test_wrap_helpers.py @@ -582,6 +582,7 @@ class TestProxyClientRefCounting: stopped: list[int] = [] monkeypatch.setattr(wrap_mod.sys, "platform", "win32") monkeypatch.setattr(wrap_mod, "_check_proxy", lambda port: port == self.PORT) + monkeypatch.setattr(wrap_mod, "_query_proxy_config", lambda port: {"pid": 123}) monkeypatch.setattr( wrap_mod, "_stop_local_proxy_for_unwrap", @@ -593,6 +594,28 @@ class TestProxyClientRefCounting: assert not proc.terminated assert stopped == [self.PORT] + def test_cleanup_uses_pre_shutdown_pid_when_health_probe_races( + self, clients_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A transient post-terminate /health miss must not orphan the listener.""" + wrap_mod._register_proxy_client(self.PORT) + proc = _FakeProxyProc() + killed: list[tuple[int, int]] = [] + monkeypatch.setattr(wrap_mod.sys, "platform", "win32") + monkeypatch.setattr(wrap_mod, "_check_proxy", lambda port: port == self.PORT) + monkeypatch.setattr(wrap_mod, "_query_proxy_config", lambda port: {"pid": 456}) + monkeypatch.setattr(wrap_mod, "_stop_local_proxy_for_unwrap", lambda port: "unidentified") + monkeypatch.setattr( + wrap_mod, + "_kill_proxy_by_pid", + lambda pid, port: killed.append((pid, port)) or True, + ) + + wrap_mod._make_cleanup([proc], self.PORT)() + + assert proc.terminated + assert killed == [(456, self.PORT)] + def test_cleanup_leaves_proxy_running_when_other_client_alive(self, clients_dir: Path) -> None: """A second live client (here: the test's parent) keeps the proxy up.""" wrap_mod._register_proxy_client(self.PORT)