legion-ref/mesh-llm
1
0
Fork 0
mirror of https://github.com/Mesh-LLM/mesh-llm.git synced 2026-08-08 22:23:19 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add multi-SDK compat smoke tests to CI (#100)

Browse source 
* Add optional OpenAI compat smoke workflow

* Run compat smoke on all PRs

* Run compat smoke on non-main PR bases

* Fix compat workflow just invocations

* Fold compat smoke into CI workflow

* Allow manual CI dispatch

* Expand compat smoke SDK coverage

* Add langchain-openai compat smoke

* Wait for routed inference before compat smokes

* Run compat smoke on split-mode mesh

---------

Co-authored-by: James Dumay <jameswdumay@gmail.com>
This commit is contained in:
Michael Neale 2026-03-31 13:27:22 +11:00 • committed by GitHub
parent 4dd53be3c4
commit 623c28f2c7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 711 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

18
.github/workflows/ci.yml vendored
View file

@ -1,10 +1,10 @@
name: CI
on:
workflow_dispatch:
push:
branches: [main]
pull_request:
branches: [main]
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -29,10 +29,19 @@ jobs:
cache: npm
cache-dependency-path: mesh-llm/ui/package-lock.json
- uses: actions/setup-python@v6
with:
python-version: "3.12"
cache: pip
cache-dependency-path: .github/workflows/ci.yml
- name: Build UI
working-directory: mesh-llm/ui
run: npm ci && npm run build
- name: Install Python SDKs
run: python -m pip install --upgrade pip openai litellm langchain-openai
# ── Rust ──
- uses: dtolnay/rust-toolchain@stable
@ -101,6 +110,13 @@ jobs:
llama.cpp/build/bin \
~/.models/$MODEL_FILE
- name: OpenAI Python compat smoke
run: |
scripts/ci-compat-smoke.sh \
target/release/mesh-llm \
llama.cpp/build/bin \
~/.models/$MODEL_FILE
- name: Split-mode test (host/worker routing)
run: |
scripts/ci-split-test.sh \

1
CONTRIBUTING.md
View file

@ -140,6 +140,7 @@ just ui-dev http://127.0.0.1:3131 5174
```bash
just stop # stop mesh/rpc/llama processes
just test # quick test against :9337
just compat-smoke ~/.models/<model>.gguf # optional 2-node + 1-client Python/Node/LiteLLM smoke
just --list # list all recipes
```

4
Justfile
View file

@ -321,6 +321,10 @@ test port="9337":
-d '{"model":"test","messages":[{"role":"user","content":"Hello! Write a haiku about distributed computing."}],"max_tokens":50}' \
| python3 -c "import sys,json; d=json.load(sys.stdin); t=d['timings']; print(d['choices'][0]['message'].get('content','')[:200]); print(f\" prompt: {t['prompt_per_second']:.1f} tok/s gen: {t['predicted_per_second']:.1f} tok/s ({t['predicted_n']} tok)\")"
# Optional SDK compatibility smoke: 2 mesh nodes + 1 lite client.
compat-smoke model:
scripts/ci-compat-smoke.sh "target/release/mesh-llm" "llama.cpp/build/bin" "{{ model }}"
# Benchmark sticky-only vs prefix-only affinity on a 3-node local mesh.
bench-prefix-affinity:
@scripts/benchmark-prefix-affinity.sh

355
scripts/ci-compat-smoke.sh Executable file
View file

@ -0,0 +1,355 @@
#!/usr/bin/env bash
# ci-compat-smoke.sh — start 2 mesh nodes + 1 lite client, then run SDK compatibility smokes.
#
# Usage: scripts/ci-compat-smoke.sh <mesh-llm-binary> <bin-dir> <model-path>
set -euo pipefail
MESH_LLM="$1"
BIN_DIR="$2"
MODEL="$3"
PYTHON_BIN="${PYTHON_BIN:-python3}"
NODE_BIN="${NODE_BIN:-node}"
NPM_BIN="${NPM_BIN:-npm}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
HOST_API_PORT=9337
HOST_CONSOLE_PORT=3131
HOST_BIND_PORT=7842
WORKER_API_PORT=9437
WORKER_CONSOLE_PORT=4131
WORKER_BIND_PORT=7843
CLIENT_API_PORT=9555
CLIENT_CONSOLE_PORT=5131
CLIENT_BIND_PORT=7844
MAX_WAIT=240
WORKDIR="$(mktemp -d)"
HOST_LOG="$WORKDIR/host.log"
WORKER_LOG="$WORKDIR/worker.log"
CLIENT_LOG="$WORKDIR/client.log"
NODE_SDK_DIR="$WORKDIR/openai-node"
echo "=== Compat Smoke Test ==="
echo " mesh-llm: $MESH_LLM"
echo " bin-dir: $BIN_DIR"
echo " model: $MODEL"
echo " workdir: $WORKDIR"
if [ ! -f "$MESH_LLM" ]; then
echo "❌ Missing mesh-llm binary: $MESH_LLM"
exit 1
fi
cleanup() {
set +e
for pid in "${CLIENT_PID:-}" "${WORKER_PID:-}" "${HOST_PID:-}"; do
if [ -n "${pid:-}" ]; then
kill "$pid" 2>/dev/null || true
pkill -P "$pid" 2>/dev/null || true
fi
done
sleep 2
for pid in "${CLIENT_PID:-}" "${WORKER_PID:-}" "${HOST_PID:-}"; do
if [ -n "${pid:-}" ]; then
kill -9 "$pid" 2>/dev/null || true
fi
done
pkill -9 -f "[/]rpc-server" 2>/dev/null || true
pkill -9 -f "[/]llama-server" 2>/dev/null || true
rm -rf "$WORKDIR"
}
trap cleanup EXIT
fail_with_logs() {
local message="$1"
echo "❌ $message"
echo "--- host log ---"
tail -80 "$HOST_LOG" 2>/dev/null || true
echo "--- worker log ---"
tail -80 "$WORKER_LOG" 2>/dev/null || true
echo "--- client log ---"
tail -80 "$CLIENT_LOG" 2>/dev/null || true
exit 1
}
assert_pid_alive() {
local pid="$1"
local name="$2"
if ! kill -0 "$pid" 2>/dev/null; then
fail_with_logs "$name exited unexpectedly"
fi
}
json_field() {
local url="$1"
local field="$2"
curl -sf "$url" | "$PYTHON_BIN" -c '
import json
import sys
field = sys.argv[1]
data = json.load(sys.stdin)
value = data
for part in field.split("."):
if part.isdigit():
value = value[int(part)]
else:
value = value.get(part)
print("" if value is None else value)
' "$field"
}
json_len() {
local url="$1"
local field="$2"
curl -sf "$url" | "$PYTHON_BIN" -c '
import json
import sys
field = sys.argv[1]
data = json.load(sys.stdin)
value = data
for part in field.split("."):
if part.isdigit():
value = value[int(part)]
else:
value = value.get(part)
print(len(value) if value is not None else 0)
' "$field"
}
wait_for_status() {
local port="$1"
local pid="$2"
local name="$3"
for i in $(seq 1 "$MAX_WAIT"); do
assert_pid_alive "$pid" "$name"
if curl -sf "http://127.0.0.1:${port}/api/status" >/dev/null 2>&1; then
return 0
fi
sleep 1
done
fail_with_logs "status endpoint on :$port for $name never came up"
}
wait_for_llama_ready() {
local port="$1"
local name="$2"
local pid="$3"
for i in $(seq 1 "$MAX_WAIT"); do
assert_pid_alive "$pid" "$name"
local ready
ready="$(json_field "http://127.0.0.1:${port}/api/status" "llama_ready" 2>/dev/null || true)"
if [ "$ready" = "True" ]; then
echo "✅ $name ready after ${i}s"
return 0
fi
if [ $((i % 20)) -eq 0 ]; then
echo " Waiting for $name model load... (${i}s)"
fi
sleep 1
done
fail_with_logs "$name did not become ready"
}
wait_for_client_mesh() {
for i in $(seq 1 "$MAX_WAIT"); do
assert_pid_alive "$CLIENT_PID" "client"
assert_pid_alive "$HOST_PID" "host"
assert_pid_alive "$WORKER_PID" "worker"
local peers
local models
peers="$(json_len "http://127.0.0.1:${CLIENT_CONSOLE_PORT}/api/status" "peers" 2>/dev/null || echo 0)"
models="$(curl -sf "http://127.0.0.1:${CLIENT_API_PORT}/v1/models" 2>/dev/null | "$PYTHON_BIN" -c 'import json,sys; print(len(json.load(sys.stdin).get("data", [])))' 2>/dev/null || echo 0)"
if [ "$peers" -ge 2 ] && [ "$models" -ge 1 ]; then
echo "✅ Client sees mesh: peers=$peers models=$models"
return 0
fi
if [ $((i % 15)) -eq 0 ]; then
echo " Waiting for client mesh visibility... (${i}s, peers=$peers, models=$models)"
fi
sleep 1
done
fail_with_logs "client never saw both mesh nodes and models"
}
wait_for_split_mesh() {
for i in $(seq 1 "$MAX_WAIT"); do
assert_pid_alive "$HOST_PID" "host"
assert_pid_alive "$WORKER_PID" "worker"
local host_is_host
local host_ready
local host_peers
local worker_is_host
local worker_ready
local worker_peers
host_is_host="$(json_field "http://127.0.0.1:${HOST_CONSOLE_PORT}/api/status" "is_host" 2>/dev/null || true)"
host_ready="$(json_field "http://127.0.0.1:${HOST_CONSOLE_PORT}/api/status" "llama_ready" 2>/dev/null || true)"
host_peers="$(json_len "http://127.0.0.1:${HOST_CONSOLE_PORT}/api/status" "peers" 2>/dev/null || echo 0)"
worker_is_host="$(json_field "http://127.0.0.1:${WORKER_CONSOLE_PORT}/api/status" "is_host" 2>/dev/null || true)"
worker_ready="$(json_field "http://127.0.0.1:${WORKER_CONSOLE_PORT}/api/status" "llama_ready" 2>/dev/null || true)"
worker_peers="$(json_len "http://127.0.0.1:${WORKER_CONSOLE_PORT}/api/status" "peers" 2>/dev/null || echo 0)"
if [ "$host_is_host" = "True" ] && [ "$host_ready" = "True" ] && [ "$host_peers" -ge 1 ]; then
echo "✅ Split mesh formed with host on :$HOST_CONSOLE_PORT after ${i}s"
return 0
fi
if [ "$worker_is_host" = "True" ] && [ "$worker_ready" = "True" ] && [ "$worker_peers" -ge 1 ]; then
echo "✅ Split mesh formed with host on :$WORKER_CONSOLE_PORT after ${i}s"
return 0
fi
if [ $((i % 15)) -eq 0 ]; then
echo " Waiting for split mesh to elect a ready host... (${i}s)"
fi
sleep 1
done
fail_with_logs "split mesh never formed a ready host"
}
wait_for_routed_inference() {
local model="$1"
for i in $(seq 1 "$MAX_WAIT"); do
assert_pid_alive "$CLIENT_PID" "client"
assert_pid_alive "$HOST_PID" "host"
assert_pid_alive "$WORKER_PID" "worker"
local response
local http_code
local body
response="$(curl -s --max-time 20 -w '\n%{http_code}' \
"http://127.0.0.1:${CLIENT_API_PORT}/v1/chat/completions" \
-H "Content-Type: application/json" \
-d "{
\"model\": \"${model}\",
\"messages\": [{\"role\": \"user\", \"content\": \"Reply with the single word ready.\"}],
\"max_tokens\": 8,
\"temperature\": 0
}" 2>/dev/null || true)"
http_code="$(printf '%s\n' "$response" | tail -n 1)"
body="$(printf '%s\n' "$response" | sed '$d')"
if [ "$http_code" = "200" ]; then
echo "✅ Routed inference ready after ${i}s"
return 0
fi
if [ "$http_code" != "503" ] && [ "$http_code" != "000" ] && [ -n "$http_code" ]; then
echo "Unexpected readiness probe response ($http_code): $body"
fail_with_logs "routed inference probe failed unexpectedly"
fi
if [ $((i % 10)) -eq 0 ]; then
echo " Waiting for routed inference readiness... (${i}s, last status=${http_code:-none})"
fi
sleep 1
done
fail_with_logs "routed inference never became ready"
}
ensure_openai_node_sdk() {
if ! command -v "$NODE_BIN" >/dev/null 2>&1; then
fail_with_logs "node is not installed"
fi
if ! command -v "$NPM_BIN" >/dev/null 2>&1; then
fail_with_logs "npm is not installed"
fi
mkdir -p "$NODE_SDK_DIR"
"$NPM_BIN" install --silent --prefix "$NODE_SDK_DIR" openai >/dev/null
}
model_id() {
curl -sf "http://127.0.0.1:${CLIENT_API_PORT}/v1/models" | "$PYTHON_BIN" -c '
import json
import sys
data = json.load(sys.stdin).get("data", [])
if not data:
raise SystemExit("no models returned")
print(data[0]["id"])
'
}
echo "Starting host..."
MESH_LLM_EPHEMERAL_KEY=1 "$MESH_LLM" \
--model "$MODEL" \
--split \
--no-draft \
--bin-dir "$BIN_DIR" \
--device CPU \
--port "$HOST_API_PORT" \
--console "$HOST_CONSOLE_PORT" \
--bind-port "$HOST_BIND_PORT" \
>"$HOST_LOG" 2>&1 &
HOST_PID=$!
wait_for_status "$HOST_CONSOLE_PORT" "$HOST_PID" "host"
TOKEN="$(json_field "http://127.0.0.1:${HOST_CONSOLE_PORT}/api/status" "token")"
if [ -z "$TOKEN" ]; then
fail_with_logs "host did not expose an invite token"
fi
wait_for_llama_ready "$HOST_CONSOLE_PORT" "host" "$HOST_PID"
echo "Starting worker..."
MESH_LLM_EPHEMERAL_KEY=1 "$MESH_LLM" \
--model "$MODEL" \
--split \
--no-draft \
--bin-dir "$BIN_DIR" \
--device CPU \
--port "$WORKER_API_PORT" \
--console "$WORKER_CONSOLE_PORT" \
--bind-port "$WORKER_BIND_PORT" \
--join "$TOKEN" \
>"$WORKER_LOG" 2>&1 &
WORKER_PID=$!
wait_for_status "$WORKER_CONSOLE_PORT" "$WORKER_PID" "worker"
wait_for_split_mesh
echo "Starting lite client..."
MESH_LLM_EPHEMERAL_KEY=1 "$MESH_LLM" \
--client \
--no-draft \
--port "$CLIENT_API_PORT" \
--console "$CLIENT_CONSOLE_PORT" \
--bind-port "$CLIENT_BIND_PORT" \
--join "$TOKEN" \
>"$CLIENT_LOG" 2>&1 &
CLIENT_PID=$!
wait_for_status "$CLIENT_CONSOLE_PORT" "$CLIENT_PID" "client"
wait_for_client_mesh
ROUTED_MODEL="$(model_id)"
echo "Using routed model: $ROUTED_MODEL"
wait_for_routed_inference "$ROUTED_MODEL"
echo "Running official openai-python smoke..."
"$PYTHON_BIN" "$REPO_ROOT/scripts/ci-openai-python-smoke.py" \
--base-url "http://127.0.0.1:${CLIENT_API_PORT}/v1"
echo "Running official openai-node smoke..."
ensure_openai_node_sdk
NODE_PATH="$NODE_SDK_DIR/node_modules" "$NODE_BIN" \
"$REPO_ROOT/scripts/ci-openai-node-smoke.cjs" \
--base-url "http://127.0.0.1:${CLIENT_API_PORT}/v1"
echo "Running LiteLLM smoke..."
"$PYTHON_BIN" "$REPO_ROOT/scripts/ci-litellm-smoke.py" \
--base-url "http://127.0.0.1:${CLIENT_API_PORT}/v1" \
--model "$ROUTED_MODEL"
echo "Running langchain-openai smoke..."
"$PYTHON_BIN" "$REPO_ROOT/scripts/ci-langchain-openai-smoke.py" \
--base-url "http://127.0.0.1:${CLIENT_API_PORT}/v1" \
--model "$ROUTED_MODEL"
echo ""
echo "=== Compat smoke passed ==="

78
scripts/ci-langchain-openai-smoke.py Normal file
View file

@ -0,0 +1,78 @@
#!/usr/bin/env python3
"""langchain-openai smoke against a mesh-llm OpenAI-compatible endpoint."""
from __future__ import annotations
import argparse
from typing import Any, Iterable
def content_text(content: Any) -> str:
if isinstance(content, str):
return content
if isinstance(content, list):
parts: list[str] = []
for item in content:
if isinstance(item, str):
parts.append(item)
elif isinstance(item, dict):
text = item.get("text")
if isinstance(text, str):
parts.append(text)
return "".join(parts)
return ""
def streamed_text(chunks: Iterable[object]) -> str:
parts: list[str] = []
saw_chunk = False
for chunk in chunks:
saw_chunk = True
text = content_text(getattr(chunk, "content", ""))
if text:
parts.append(text)
if not saw_chunk:
raise RuntimeError("stream returned no chunks")
text = "".join(parts).strip()
if not text:
raise RuntimeError("stream returned no content")
return text
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--model", required=True)
args = parser.parse_args()
try:
from langchain_openai import ChatOpenAI
except ModuleNotFoundError as exc:
raise SystemExit(
"langchain-openai package not installed; run `python -m pip install langchain-openai` first"
) from exc
llm = ChatOpenAI(
model=args.model,
api_key="mesh-llm-ci",
base_url=args.base_url,
temperature=0,
max_tokens=32,
stream_usage=False,
)
print(f"Using model: {args.model}")
response = llm.invoke([("human", "Say hello in exactly 4 words.")])
message = content_text(response.content).strip()
if not message:
raise RuntimeError("non-streaming chat returned empty content")
print(f"Non-streaming response: {message}")
stream = llm.stream([("human", "Count from one to three.")])
text = streamed_text(stream)
print(f"Streaming response: {text}")
if __name__ == "__main__":
main()

85
scripts/ci-litellm-smoke.py Normal file
View file

@ -0,0 +1,85 @@
#!/usr/bin/env python3
"""LiteLLM smoke against a mesh-llm OpenAI-compatible endpoint."""
from __future__ import annotations
import argparse
from typing import Any, Iterable
def get_field(value: Any, name: str) -> Any:
if isinstance(value, dict):
return value.get(name)
return getattr(value, name, None)
def streamed_text(chunks: Iterable[object]) -> str:
parts: list[str] = []
saw_choice = False
for chunk in chunks:
choices = get_field(chunk, "choices") or []
for choice in choices:
saw_choice = True
delta = get_field(choice, "delta")
if delta is None:
continue
content = get_field(delta, "content")
if isinstance(content, str) and content:
parts.append(content)
if not saw_choice:
raise RuntimeError("stream returned no choices")
text = "".join(parts).strip()
if not text:
raise RuntimeError("stream returned no content")
return text
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--model", required=True)
args = parser.parse_args()
try:
from litellm import completion
except ModuleNotFoundError as exc:
raise SystemExit(
"litellm package not installed; run `python -m pip install litellm` first"
) from exc
provider_model = f"openai/{args.model}"
print(f"Using model: {provider_model}")
response = completion(
model=provider_model,
api_base=args.base_url,
api_key="mesh-llm-ci",
messages=[
{"role": "user", "content": "Say hello in exactly 4 words."},
],
max_tokens=32,
temperature=0,
)
first_choice = (get_field(response, "choices") or [None])[0]
message = get_field(get_field(first_choice, "message"), "content")
if not isinstance(message, str) or not message.strip():
raise RuntimeError("non-streaming chat returned empty content")
print(f"Non-streaming response: {message.strip()}")
stream = completion(
model=provider_model,
api_base=args.base_url,
api_key="mesh-llm-ci",
messages=[
{"role": "user", "content": "Count from one to three."},
],
max_tokens=32,
temperature=0,
stream=True,
)
text = streamed_text(stream)
print(f"Streaming response: {text}")
if __name__ == "__main__":
main()

90
scripts/ci-openai-node-smoke.cjs Normal file
View file

@ -0,0 +1,90 @@
#!/usr/bin/env node
/* Official openai-node smoke against a mesh-llm OpenAI-compatible endpoint. */
const OpenAI = require('openai');
function parseArgs(argv) {
const args = {};
for (let i = 2; i < argv.length; i += 1) {
const arg = argv[i];
if (arg === '--base-url') {
args.baseUrl = argv[++i];
} else {
throw new Error(`unknown argument: ${arg}`);
}
}
if (!args.baseUrl) {
throw new Error('--base-url is required');
}
return args;
}
async function streamedText(stream) {
const parts = [];
let sawChoice = false;
for await (const chunk of stream) {
const choices = chunk.choices || [];
for (const choice of choices) {
sawChoice = true;
const content = choice?.delta?.content;
if (typeof content === 'string' && content.length > 0) {
parts.push(content);
}
}
}
if (!sawChoice) {
throw new Error('stream returned no choices');
}
const text = parts.join('').trim();
if (!text) {
throw new Error('stream returned no content');
}
return text;
}
async function main() {
const args = parseArgs(process.argv);
const client = new OpenAI({
apiKey: 'mesh-llm-ci',
baseURL: args.baseUrl,
});
const models = await client.models.list();
if (!models.data.length) {
throw new Error('models.list returned no models');
}
const model = models.data[0].id;
console.log(`Using model: ${model}`);
const response = await client.chat.completions.create({
model,
messages: [{ role: 'user', content: 'Say hello in exactly 4 words.' }],
max_tokens: 32,
temperature: 0,
});
const message = response.choices?.[0]?.message?.content?.trim();
if (!message) {
throw new Error('non-streaming chat returned empty content');
}
console.log(`Non-streaming response: ${message}`);
const stream = await client.chat.completions.create({
model,
messages: [{ role: 'user', content: 'Count from one to three.' }],
max_tokens: 32,
temperature: 0,
stream: true,
});
const text = await streamedText(stream);
console.log(`Streaming response: ${text}`);
}
main().catch((error) => {
console.error(error instanceof Error ? error.stack || error.message : String(error));
process.exit(1);
});

81
scripts/ci-openai-python-smoke.py Executable file
View file

@ -0,0 +1,81 @@
#!/usr/bin/env python3
"""Official openai-python smoke against a mesh-llm OpenAI-compatible endpoint."""
from __future__ import annotations
import argparse
from typing import Iterable
def streamed_text(chunks: Iterable[object]) -> str:
parts: list[str] = []
saw_choice = False
for chunk in chunks:
choices = getattr(chunk, "choices", None) or []
for choice in choices:
saw_choice = True
delta = getattr(choice, "delta", None)
if delta is None:
continue
content = getattr(delta, "content", None)
if content:
parts.append(content)
if not saw_choice:
raise RuntimeError("stream returned no choices")
text = "".join(parts).strip()
if not text:
raise RuntimeError("stream returned no content")
return text
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
args = parser.parse_args()
try:
from openai import OpenAI
except ModuleNotFoundError as exc:
raise SystemExit(
"openai package not installed; run `python -m pip install openai` first"
) from exc
client = OpenAI(
api_key="mesh-llm-ci",
base_url=args.base_url,
)
models = client.models.list()
if not models.data:
raise RuntimeError("models.list returned no models")
model = models.data[0].id
print(f"Using model: {model}")
response = client.chat.completions.create(
model=model,
messages=[
{"role": "user", "content": "Say hello in exactly 4 words."},
],
max_tokens=32,
temperature=0,
)
message = response.choices[0].message.content
if not message or not message.strip():
raise RuntimeError("non-streaming chat returned empty content")
print(f"Non-streaming response: {message.strip()}")
stream = client.chat.completions.create(
model=model,
messages=[
{"role": "user", "content": "Count from one to three."},
],
max_tokens=32,
temperature=0,
stream=True,
)
text = streamed_text(stream)
print(f"Streaming response: {text}")
if __name__ == "__main__":
main()