diff --git a/e2e/wrap/run.py b/e2e/wrap/run.py index 21d06b2ca..2a6c93544 100644 --- a/e2e/wrap/run.py +++ b/e2e/wrap/run.py @@ -21,6 +21,11 @@ REPO_ROOT = Path("/workspace") PLUGIN_DIR = REPO_ROOT / "plugins" / "openclaw" SDK_DIR = REPO_ROOT / "sdk" / "typescript" RTK_MARKER = "" +PROXY_PORT = 28887 +CODEX_PORT = 28888 +AIDER_PORT = 28889 +CURSOR_PORT = 28890 +OPENCLAW_PROXY_PORT = 28891 def log(message: str) -> None: @@ -199,6 +204,7 @@ def create_shims(shim_dir: Path) -> None: import json import os import sys + import urllib.request from pathlib import Path tool = Path(sys.argv[0]).name @@ -214,6 +220,27 @@ def create_shims(shim_dir: Path) -> None: if os.environ.get(key) is not None }, } + + probes = [] + + def fetch(url: str, *, headers: dict[str, str] | None = None) -> None: + request = urllib.request.Request(url, headers=headers or {}) + with urllib.request.urlopen(request, timeout=10) as response: + probes.append({"url": url, "status": response.status}) + + openai_base = os.environ.get("OPENAI_BASE_URL") or os.environ.get("OPENAI_API_BASE") + if openai_base: + fetch( + f"{openai_base.rstrip('/')}/models", + headers={"Authorization": "Bearer test-key"}, + ) + + anthropic_base = os.environ.get("ANTHROPIC_BASE_URL") + if anthropic_base: + fetch(f"{anthropic_base.rstrip('/')}/health") + + record["probes"] = probes + with (log_dir / f"{tool}.jsonl").open("a", encoding="utf-8") as handle: handle.write(json.dumps(record) + "\\n") print(f"{tool} shim executed") @@ -234,131 +261,9 @@ def create_shims(shim_dir: Path) -> None: raise SystemExit(0) """ ) - openclaw_shim = textwrap.dedent( - """\ - #!/usr/bin/env python3 - from __future__ import annotations - - import json - import os - import sys - from pathlib import Path - - state_path = Path(os.environ["HEADROOM_E2E_OPENCLAW_STATE"]) - config_path = Path(os.environ["HEADROOM_E2E_OPENCLAW_CONFIG"]) - log_dir = Path(os.environ["HEADROOM_E2E_LOG_DIR"]) - log_dir.mkdir(parents=True, exist_ok=True) - - def default_state() -> dict: - return { - "plugins": { - "entries": {}, - "slots": {"contextEngine": "legacy"}, - }, - "installs": [], - "gateway_actions": [], - } - - def load_state() -> dict: - if state_path.exists(): - return json.loads(state_path.read_text(encoding="utf-8")) - state = default_state() - save_state(state) - return state - - def save_state(state: dict) -> None: - state_path.parent.mkdir(parents=True, exist_ok=True) - state_path.write_text(json.dumps(state, indent=2) + "\\n", encoding="utf-8") - config_path.parent.mkdir(parents=True, exist_ok=True) - config_path.write_text(json.dumps(state, indent=2) + "\\n", encoding="utf-8") - - def log_call(args: list[str]) -> None: - record = { - "tool": "openclaw", - "argv": args, - "cwd": os.getcwd(), - } - with (log_dir / "openclaw.jsonl").open("a", encoding="utf-8") as handle: - handle.write(json.dumps(record) + "\\n") - - def get_nested(container: dict, dotted_path: str): - current = container - for key in dotted_path.split("."): - if not isinstance(current, dict) or key not in current: - return None - current = current[key] - return current - - def set_nested(container: dict, dotted_path: str, value) -> None: - keys = dotted_path.split(".") - current = container - for key in keys[:-1]: - current = current.setdefault(key, {}) - current[keys[-1]] = value - - args = sys.argv[1:] - if not args or args[0] in {"--help", "-h", "help"}: - print("openclaw shim") - raise SystemExit(0) - - log_call(args) - state = load_state() - extensions_dir = config_path.parent / "extensions" - - if args[:2] == ["config", "file"]: - config_path.parent.mkdir(parents=True, exist_ok=True) - config_path.touch() - print(str(config_path)) - raise SystemExit(0) - - if args[:2] == ["config", "get"]: - print(json.dumps(get_nested(state, args[2]))) - raise SystemExit(0) - - if args[:2] == ["config", "set"]: - set_nested(state, args[2], json.loads(args[3])) - save_state(state) - print("ok") - raise SystemExit(0) - - if args[:2] == ["config", "validate"]: - print("valid") - raise SystemExit(0) - - if args[:2] == ["gateway", "restart"] or args[:2] == ["gateway", "start"]: - state["gateway_actions"].append(args[1]) - save_state(state) - print(args[1]) - raise SystemExit(0) - - if args[:2] == ["plugins", "install"]: - state["installs"].append({"argv": args, "cwd": os.getcwd()}) - save_state(state) - (extensions_dir / "headroom").mkdir(parents=True, exist_ok=True) - print("installed") - raise SystemExit(0) - - if args[:3] == ["plugins", "inspect", "headroom"]: - print( - json.dumps( - { - "id": "headroom", - "installed": True, - "entry": get_nested(state, "plugins.entries.headroom"), - } - ) - ) - raise SystemExit(0) - - print(f"unsupported openclaw args: {args}", file=sys.stderr) - raise SystemExit(2) - """ - ) - write_executable(shim_dir / "codex", generic_shim) write_executable(shim_dir / "aider", generic_shim) write_executable(shim_dir / "rtk", rtk_shim) - write_executable(shim_dir / "openclaw", openclaw_shim) def start_mock_server(port: int) -> tuple[MockOpenAIServer, threading.Thread]: @@ -394,6 +299,62 @@ def stop_process(proc: subprocess.Popen[str]) -> None: proc.communicate(timeout=5) +def wait_for_command_success( + cmd: list[str], + *, + env: dict[str, str], + cwd: Path | None = None, + timeout: int = 30, +) -> subprocess.CompletedProcess[str]: + deadline = time.time() + timeout + last_output = "" + while time.time() < deadline: + remaining = deadline - time.time() + per_call_timeout = max(1.0, min(5.0, remaining)) + try: + result = subprocess.run( + cmd, + env=env, + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=per_call_timeout, + ) + except subprocess.TimeoutExpired: + last_output = f"Command timed out after {per_call_timeout:.1f}s" + time.sleep(1) + continue + if result.returncode == 0: + if result.stdout.strip(): + print(result.stdout.rstrip(), flush=True) + if result.stderr.strip(): + print(result.stderr.rstrip(), file=sys.stderr, flush=True) + return result + last_output = "\n".join( + part for part in (result.stdout.strip(), result.stderr.strip()) if part + ) + time.sleep(1) + raise RuntimeError( + f"Timed out waiting for command to succeed: {' '.join(cmd)}\nLast output:\n{last_output}" + ) + + +def start_openclaw_gateway(env: dict[str, str], cwd: Path) -> subprocess.Popen[str]: + log("Starting OpenClaw gateway for e2e verification") + return subprocess.Popen( + ["openclaw", "gateway"], + env=env, + cwd=str(cwd), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + ) + + def verify_installs() -> None: log("Verifying installed packages and binaries") for tool in ("headroom", "codex", "aider", "openclaw"): @@ -430,7 +391,7 @@ def prepare_local_openclaw_plugin(base_env: dict[str, str], tmp_dir: Path) -> Pa def verify_proxy_round_trip(base_env: dict[str, str], mock_server: MockOpenAIServer) -> None: - proxy_port = 18787 + proxy_port = PROXY_PORT proc = start_proxy(proxy_port, base_env) try: health = wait_for_http(f"http://127.0.0.1:{proxy_port}/health") @@ -471,7 +432,7 @@ def verify_proxy_round_trip(base_env: dict[str, str], mock_server: MockOpenAISer def verify_codex_wrap(base_env: dict[str, str], project_dir: Path, log_dir: Path) -> None: - port = 18788 + port = CODEX_PORT run( ["headroom", "wrap", "codex", "--port", str(port), "--", "--help"], env=base_env, @@ -494,10 +455,14 @@ def verify_codex_wrap(base_env: dict[str, str], project_dir: Path, log_dir: Path env_vars.get("OPENAI_BASE_URL") == f"http://127.0.0.1:{port}/v1", "Codex wrap should set OPENAI_BASE_URL", ) + assert_true( + entries[-1]["probes"] == [{"url": f"http://127.0.0.1:{port}/v1/models", "status": 200}], + "Codex shim should prove OPENAI_BASE_URL points at a live proxy", + ) def verify_aider_wrap(base_env: dict[str, str], project_dir: Path, log_dir: Path) -> None: - port = 18789 + port = AIDER_PORT run( ["headroom", "wrap", "aider", "--port", str(port), "--", "--help"], env=base_env, @@ -522,10 +487,18 @@ def verify_aider_wrap(base_env: dict[str, str], project_dir: Path, log_dir: Path env_vars.get("ANTHROPIC_BASE_URL") == f"http://127.0.0.1:{port}", "Aider wrap should set ANTHROPIC_BASE_URL", ) + assert_true( + entries[-1]["probes"] + == [ + {"url": f"http://127.0.0.1:{port}/v1/models", "status": 200}, + {"url": f"http://127.0.0.1:{port}/health", "status": 200}, + ], + "Aider shim should prove both configured base URLs point at a live proxy", + ) def verify_cursor_wrap(base_env: dict[str, str], project_dir: Path) -> None: - port = 18790 + port = CURSOR_PORT proc = subprocess.Popen( ["headroom", "wrap", "cursor", "--port", str(port)], env=base_env, @@ -542,6 +515,10 @@ def verify_cursor_wrap(base_env: dict[str, str], project_dir: Path) -> None: f"http://127.0.0.1:{port}/v1" in output, "Cursor wrap should print the OpenAI base URL override", ) + assert_true( + f"http://127.0.0.1:{port}" in output, + "Cursor wrap should print the Anthropic base URL override", + ) wait_for_http(f"http://127.0.0.1:{port}/health", timeout=15) cursorrules = project_dir / ".cursorrules" assert_true(cursorrules.exists(), "Cursor wrap should create .cursorrules") @@ -557,8 +534,9 @@ def verify_openclaw_wrap( base_env: dict[str, str], project_dir: Path, plugin_dir: Path, - state_path: Path, ) -> None: + port = OPENCLAW_PROXY_PORT + gateway_proc: subprocess.Popen[str] | None = None run( [ "headroom", @@ -567,7 +545,7 @@ def verify_openclaw_wrap( "--plugin-path", str(plugin_dir), "--proxy-port", - "18791", + str(port), "--startup-timeout-ms", "5000", "--verbose", @@ -579,19 +557,66 @@ def verify_openclaw_wrap( dist_index = plugin_dir / "dist" / "index.js" assert_true(dist_index.exists(), "OpenClaw plugin build should produce dist/index.js") - state = json.loads(state_path.read_text(encoding="utf-8")) - entry = state["plugins"]["entries"]["headroom"] - assert_true(entry["enabled"] is True, "OpenClaw wrap should enable the plugin") - assert_true(entry["config"]["proxyPort"] == 18791, "OpenClaw wrap should set proxy port") - assert_true(entry["config"]["autoStart"] is True, "OpenClaw wrap should enable autoStart") - assert_true( - state["plugins"]["slots"]["contextEngine"] == "headroom", - "OpenClaw wrap should set the context engine slot", - ) - assert_true( - state["gateway_actions"] == ["restart"], - "OpenClaw wrap should restart the gateway once", - ) + config_file = run(["openclaw", "config", "file"], env=base_env, cwd=project_dir, timeout=60) + config_path_str = config_file.stdout.strip().splitlines()[-1].strip() + if config_path_str.startswith("~/"): + config_path = Path(base_env["HOME"]) / config_path_str[2:] + else: + config_path = Path(config_path_str) + assert_true(config_path.exists(), "OpenClaw should create a config file") + + state = json.loads(config_path.read_text(encoding="utf-8")) + if state.get("gateway", {}).get("mode") != "local": + run( + [ + "openclaw", + "config", + "set", + "gateway.mode", + json.dumps("local"), + "--strict-json", + ], + env=base_env, + cwd=project_dir, + timeout=60, + ) + state = json.loads(config_path.read_text(encoding="utf-8")) + + try: + try: + wait_for_command_success( + ["openclaw", "health"], env=base_env, cwd=project_dir, timeout=5 + ) + except RuntimeError: + gateway_proc = start_openclaw_gateway(base_env, project_dir) + try: + wait_for_command_success( + ["openclaw", "health"], env=base_env, cwd=project_dir, timeout=30 + ) + except RuntimeError as exc: + gateway_output = "" + if gateway_proc.stdout is not None: + gateway_output = gateway_proc.stdout.read() + raise RuntimeError(f"{exc}\nGateway output:\n{gateway_output}") from exc + + entry = state["plugins"]["entries"]["headroom"] + assert_true(entry["enabled"] is True, "OpenClaw wrap should enable the plugin") + assert_true(entry["config"]["proxyPort"] == port, "OpenClaw wrap should set proxy port") + assert_true( + entry["config"].get("autoStart", True) is True, + "OpenClaw wrap should leave autoStart enabled", + ) + assert_true( + state["gateway"]["mode"] == "local", + "OpenClaw e2e bootstrap should set gateway.mode=local", + ) + assert_true( + state["plugins"]["slots"]["contextEngine"] == "headroom", + "OpenClaw wrap should set the context engine slot", + ) + finally: + if gateway_proc is not None: + stop_process(gateway_proc) def main() -> None: @@ -602,8 +627,6 @@ def main() -> None: project_dir = tmp_dir / "project" shim_dir = tmp_dir / "shim-bin" log_dir = tmp_dir / "logs" - openclaw_state_path = tmp_dir / "openclaw" / "state.json" - openclaw_config_path = tmp_dir / "openclaw" / "config" / "config.json" for path in (home_dir, project_dir, shim_dir, log_dir): path.mkdir(parents=True, exist_ok=True) @@ -616,8 +639,6 @@ def main() -> None: "HOME": str(home_dir), "PATH": f"{shim_dir}{os.pathsep}{base_env['PATH']}", "HEADROOM_E2E_LOG_DIR": str(log_dir), - "HEADROOM_E2E_OPENCLAW_STATE": str(openclaw_state_path), - "HEADROOM_E2E_OPENCLAW_CONFIG": str(openclaw_config_path), "OPENAI_TARGET_API_URL": "http://127.0.0.1:19001/v1", } ) @@ -628,7 +649,7 @@ def main() -> None: verify_aider_wrap(base_env, project_dir, log_dir) verify_cursor_wrap(base_env, project_dir) local_plugin_dir = prepare_local_openclaw_plugin(base_env, tmp_dir) - verify_openclaw_wrap(base_env, project_dir, local_plugin_dir, openclaw_state_path) + verify_openclaw_wrap(base_env, project_dir, local_plugin_dir) finally: mock_server.shutdown() mock_thread.join(timeout=5) diff --git a/headroom/evals/memory/runner_v2.py b/headroom/evals/memory/runner_v2.py index e42ff56cb..63434e6dc 100644 --- a/headroom/evals/memory/runner_v2.py +++ b/headroom/evals/memory/runner_v2.py @@ -15,9 +15,9 @@ from __future__ import annotations import asyncio import json import logging +import tempfile import time import uuid -import tempfile from collections.abc import Callable from dataclasses import dataclass, field from datetime import datetime diff --git a/headroom/evals/memory/runner_v3.py b/headroom/evals/memory/runner_v3.py index 91e49f421..08f793325 100644 --- a/headroom/evals/memory/runner_v3.py +++ b/headroom/evals/memory/runner_v3.py @@ -20,9 +20,9 @@ from __future__ import annotations import asyncio import json import logging +import tempfile import time import uuid -import tempfile from dataclasses import dataclass, field from datetime import datetime from pathlib import Path diff --git a/headroom/rtk/installer.py b/headroom/rtk/installer.py index 3336418bf..e7fb5e420 100644 --- a/headroom/rtk/installer.py +++ b/headroom/rtk/installer.py @@ -5,7 +5,6 @@ from __future__ import annotations import io import logging import platform -import ssl import stat import subprocess import tarfile @@ -77,20 +76,17 @@ def download_rtk(version: str | None = None) -> Path: # Validate URL scheme to prevent B310 warning if not url.startswith(("http://", "https://")): raise ValueError(f"Invalid URL scheme in {url}") - - # Try default SSL first, fall back to unverified for macOS framework Python + + # Fail closed on TLS errors rather than executing an unverifiable download. try: with urlopen(url, timeout=30) as response: data = response.read() - except Exception as ssl_err: - if "CERTIFICATE_VERIFY_FAILED" in str(ssl_err): - ctx = ssl.create_default_context() - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - with urlopen(url, timeout=30, context=ctx) as response: - data = response.read() - else: - raise + except Exception as download_err: + if "CERTIFICATE_VERIFY_FAILED" in str(download_err): + raise RuntimeError( + "TLS verification failed downloading rtk; fix the local trust store and retry." + ) from download_err + raise except Exception as e: raise RuntimeError(f"Failed to download rtk from {url}: {e}") from e diff --git a/tests/test_compression_cache.py b/tests/test_compression_cache.py index 6e9f14caf..5d0ec8640 100644 --- a/tests/test_compression_cache.py +++ b/tests/test_compression_cache.py @@ -184,9 +184,7 @@ class TestCompressionCacheFrozenCount: {"role": "user", "content": "hello"}, { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": tool_content} - ], + "content": [{"type": "tool_result", "tool_use_id": "t1", "content": tool_content}], }, {"role": "user", "content": "follow up"}, ] @@ -203,9 +201,7 @@ class TestCompressionCacheFrozenCount: {"role": "user", "content": "hi"}, { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": tool_content} - ], + "content": [{"type": "tool_result", "tool_use_id": "t1", "content": tool_content}], }, ] # Compressed is identical to originals (no compression happened) @@ -213,9 +209,7 @@ class TestCompressionCacheFrozenCount: {"role": "user", "content": "hi"}, { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": tool_content} - ], + "content": [{"type": "tool_result", "tool_use_id": "t1", "content": tool_content}], }, ] cache.update_from_result(originals, compressed) @@ -228,9 +222,7 @@ class TestCompressionCacheFrozenCount: {"role": "user", "content": "hello"}, { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": tool_content} - ], + "content": [{"type": "tool_result", "tool_use_id": "t1", "content": tool_content}], }, {"role": "user", "content": "more stuff"}, ] @@ -244,15 +236,11 @@ class TestCompressionCacheFrozenCount: {"role": "user", "content": "hi"}, { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": content_a} - ], + "content": [{"type": "tool_result", "tool_use_id": "t1", "content": content_a}], }, { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "t2", "content": content_b} - ], + "content": [{"type": "tool_result", "tool_use_id": "t2", "content": content_b}], }, ] # Mark first 2 messages (msg[0] + msg[1])