fix(moa): signal all-workers-fail as a proper error response

PR #566 review feedback (Apr 2026):

> One concurrency request returned HTTP 200 even though the response
> body said all MoA workers failed. That's a bad client contract.
> If all workers fail, the API should probably return a proper error,
> not a successful-looking response with failure text inside it.

The MoA gateway was returning a body shaped identically to a
successful `chat.completion` with the error string smuggled into
`choices[0].message.content` and `finish_reason: "stop"`. The
ingress wrapped that body in an HTTP 200. A client checking either
the HTTP status, the top-level `error` field, or `finish_reason`
saw "success."

## Test (added first, observed failing)

`crates/mesh-mixture-of-agents/tests/sim_all_workers_fail.rs` drives
`moa::handle_turn` with three `AlwaysErrBackend`s, asserts the
result body is distinguishable from a successful `chat.completion`
\u2014 either the top-level `object` is not `chat.completion`, or there
is a top-level `error`, or `finish_reason` is one of `error` /
`moa_failed`.

The test fails against the pre-fix gateway with output

> object=Some("chat.completion"), finish_reason=Some("stop"),
> has top-level error=false

## Fix

* `mesh-mixture-of-agents/src/lib.rs` \u2014 `error_response()` now
  attaches a top-level OpenAI-shape `error` object and emits
  `finish_reason: "error"`. The error text stays in `content`
  for unstructured clients.
* `mesh-llm-host-runtime/src/network/openai/transport.rs` \u2014 new
  `send_json_with_status_and_headers()` helper for sending a custom
  status code with a full structured body and observability headers.
* `mesh-llm-host-runtime/src/network/openai/moa_gateway.rs` \u2014
  `write_moa_response` now takes the full `TurnResult` and sends
  HTTP 502 (Bad Gateway) when `turn_kind == Failed` for non-streaming
  responses. Streaming SSE stays 200 because we can't change the
  status after the headers are sent; the failure rides in the chunked
  body (which now carries the structured error).

## Validation

`cargo test -p mesh-mixture-of-agents` \u2014 70 unit + 1 new
integration test pass.

`cargo test -p mesh-llm-host-runtime --lib` \u2014 1434/1434 pass.

`cargo clippy -p mesh-mixture-of-agents --all-targets -- -D warnings`
\u2014 clean.

`cargo clippy -p mesh-llm-host-runtime --all-targets -- -D warnings`
\u2014 clean.

`cargo fmt --all -- --check` \u2014 clean.

crates/mesh-llm-host-runtime/src/network/openai/moa_gateway.rs

@@ -66,25 +66,32 @@ async fn run_moa_turn(
 
     let moa_result = moa::handle_turn(config, &moa_body).await;
     let extra_headers = build_moa_headers(&moa_result);
-    write_moa_response(
-        tcp_stream,
-        &moa_result.response_body,
-        &extra_headers,
-        was_streaming,
-    )
-    .await;
+    write_moa_response(tcp_stream, &moa_result, &extra_headers, was_streaming).await;
 }
 
 /// Write the MoA response on the chosen transport (JSON or SSE), logging
 /// (but not propagating) any I/O error.
+///
+/// When the gateway reports `TurnKind::Failed` we send an HTTP 502 (Bad
+/// Gateway) with the structured error body, rather than HTTP 200. The
+/// crate's `error_response` already carries an OpenAI-shape top-level
+/// `error` object and `finish_reason: "error"`, but unsophisticated
+/// clients that only check the HTTP status need that status to actually
+/// reflect failure.
 async fn write_moa_response(
     tcp_stream: TcpStream,
-    body: &serde_json::Value,
+    moa_result: &moa::TurnResult,
     extra_headers: &[(&str, String)],
     was_streaming: bool,
 ) {
+    let body = &moa_result.response_body;
     let result = if was_streaming {
+        // SSE always uses 200: the failure signal rides in the streamed
+        // chunks (the body includes `error` and `finish_reason: "error"`).
+        // Once the headers are sent we cannot change the status code.
         send_moa_as_sse(tcp_stream, body, extra_headers).await
+    } else if moa_result.turn_kind == moa::TurnKind::Failed {
+        proxy::send_json_with_status_and_headers(tcp_stream, 502, body, extra_headers).await
     } else {
         proxy::send_json_ok_with_headers(tcp_stream, body, extra_headers).await
     };

crates/mesh-llm-host-runtime/src/network/openai/transport.rs

@@ -4013,6 +4013,47 @@ pub async fn send_json_ok_with_headers(
     Ok(())
 }
 
+/// Send a JSON body with a non-200 status and the given extra headers.
+///
+/// The body is sent verbatim — caller controls the shape. Use for cases
+/// where the in-band payload is already a structured error (e.g. MoA's
+/// `error_response`) and we still want to attach observability headers
+/// while signalling failure via the HTTP status line.
+pub async fn send_json_with_status_and_headers(
+    mut stream: TcpStream,
+    code: u16,
+    data: &serde_json::Value,
+    extra_headers: &[(&str, String)],
+) -> std::io::Result<()> {
+    let status = match code {
+        400 => "Bad Request",
+        404 => "Not Found",
+        409 => "Conflict",
+        422 => "Unprocessable Content",
+        429 => "Too Many Requests",
+        500 => "Internal Server Error",
+        502 => "Bad Gateway",
+        503 => "Service Unavailable",
+        504 => "Gateway Timeout",
+        _ => "Error",
+    };
+    let body = data.to_string();
+    let mut headers = format!("HTTP/1.1 {code} {status}\r\nContent-Type: application/json\r\n");
+    for (name, value) in extra_headers {
+        // Strip CR/LF defensively against header-injection.
+        let safe_value: String = value.chars().filter(|c| *c != '\r' && *c != '\n').collect();
+        headers.push_str(name);
+        headers.push_str(": ");
+        headers.push_str(&safe_value);
+        headers.push_str("\r\n");
+    }
+    headers.push_str(&format!("Content-Length: {}\r\n\r\n", body.len()));
+    stream.write_all(headers.as_bytes()).await?;
+    stream.write_all(body.as_bytes()).await?;
+    stream.shutdown().await?;
+    Ok(())
+}
+
 pub async fn send_400(mut stream: TcpStream, msg: &str) -> std::io::Result<()> {
     let body = serde_json::to_vec(&serde_json::json!({ "error": msg }))
         .expect("serializing JSON error response should not fail");

crates/mesh-mixture-of-agents/src/lib.rs

@@ -457,15 +457,34 @@ fn best_answer(outputs: &[WorkerOutput]) -> String {
         .unwrap_or_default()
 }
 
+/// Build a response body that signals MoA-level failure to the client.
+///
+/// Distinguishable from a successful `chat.completion` in three ways:
+///
+///   * Top-level `error` object (OpenAI error-shape) so SDKs that read
+///     `response.error` see the failure without parsing `choices`.
+///   * `choices[0].finish_reason == "error"` (instead of `"stop"`) so
+///     SDKs that branch on `finish_reason` see the failure too.
+///   * The error text is still placed in `choices[0].message.content`
+///     so unstructured clients still surface a useful string to the
+///     human, just not as a successful assistant reply.
+///
+/// The ingress layer is responsible for choosing the HTTP status; this
+/// body is the in-band signal.
 fn error_response(message: &str) -> Value {
     json!({
         "id": format!("chatcmpl-moa-{}", short_id()),
         "object": "chat.completion",
         "model": VIRTUAL_MODEL_NAME,
+        "error": {
+            "message": message,
+            "type": "moa_failure",
+            "code": "all_workers_failed",
+        },
         "choices": [{
             "index": 0,
             "message": { "role": "assistant", "content": message },
-            "finish_reason": "stop"
+            "finish_reason": "error"
         }],
         "usage": { "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0 }
     })

crates/mesh-mixture-of-agents/tests/sim_all_workers_fail.rs

@@ -0,0 +1,159 @@
+//! Simulated-mesh integration test for the "all workers failed" path.
+//!
+//! Pin the client contract for the case where every fan-out worker
+//! fails: the MoA gateway must return a response body that a client
+//! can _distinguish_ from a successful chat completion without parsing
+//! the model's free-form output.
+//!
+//! Background — PR #566 review feedback (Apr 2026):
+//!
+//! > One concurrency request returned HTTP 200 even though the response
+//! > body said all MoA workers failed. That's a bad client contract.
+//! > If all workers fail, the API should probably return a proper
+//! > error, not a successful-looking response with failure text inside
+//! > it.
+//!
+//! Today `handle_turn` returns a body shaped like a successful
+//! `chat.completion` with the error text in `choices[0].message.content`
+//! and `finish_reason: "stop"`. There is no top-level `error` field, no
+//! HTTP status carried in-band, and no way for an unsophisticated client
+//! to know the call failed without string-matching the assistant text.
+//!
+//! This test drives `handle_turn` with three mock backends that all
+//! error, then asserts the contract we want clients to be able to rely
+//! on. It is expected to fail against the current implementation — that
+//! failure is what we then fix.
+
+use async_trait::async_trait;
+use mesh_mixture_of_agents as moa;
+use serde_json::{json, Value};
+use std::sync::Arc;
+use std::time::Duration;
+
+/// Backend that returns a configured error on every call. Counts calls
+/// per model so a test can also assert the dispatch fanned out as
+/// expected.
+struct AlwaysErrBackend {
+    err: String,
+    calls: std::sync::atomic::AtomicUsize,
+}
+
+impl AlwaysErrBackend {
+    fn new(err: impl Into<String>) -> Arc<Self> {
+        Arc::new(Self {
+            err: err.into(),
+            calls: std::sync::atomic::AtomicUsize::new(0),
+        })
+    }
+}
+
+#[async_trait]
+impl moa::ModelBackend for AlwaysErrBackend {
+    async fn chat_completion(
+        &self,
+        _model: &str,
+        _messages: &[Value],
+        _tools: Option<&Value>,
+        _max_tokens: u32,
+        _timeout: Duration,
+        _sampling: moa::SamplingParams,
+    ) -> Result<Value, String> {
+        self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        Err(self.err.clone())
+    }
+}
+
+fn three_failing_backends() -> moa::GatewayConfig {
+    let alpha = AlwaysErrBackend::new("HTTP 502 from peer alpha");
+    let beta = AlwaysErrBackend::new("connect timeout to peer beta");
+    let gamma = AlwaysErrBackend::new("stream closed unexpectedly from peer gamma");
+
+    let backends: Vec<Arc<dyn moa::ModelBackend>> =
+        vec![alpha.clone(), beta.clone(), gamma.clone()];
+    let models = vec![
+        moa::ModelEntry {
+            name: "alpha-3b".into(),
+            backend_index: 0,
+        },
+        moa::ModelEntry {
+            name: "beta-13b".into(),
+            backend_index: 1,
+        },
+        moa::ModelEntry {
+            name: "gamma-32b".into(),
+            backend_index: 2,
+        },
+    ];
+
+    moa::GatewayConfig {
+        backends,
+        models,
+        worker_timeout: Duration::from_secs(2),
+        hedge_delay: Duration::from_millis(200),
+        reducer_timeout: Duration::from_secs(2),
+    }
+}
+
+fn user_turn(content: &str) -> Value {
+    json!({
+        "model": "mesh",
+        "messages": [{"role": "user", "content": content}],
+        "max_tokens": 64,
+    })
+}
+
+#[tokio::test]
+async fn all_workers_fail_returns_distinguishable_error_body() {
+    let config = three_failing_backends();
+    let body = user_turn("What is the capital of Japan?");
+
+    let result = moa::handle_turn(&config, &body).await;
+
+    // First, lock in the path classification we already have.
+    assert_eq!(
+        result.turn_kind,
+        moa::TurnKind::Failed,
+        "all-workers-fail must classify as TurnKind::Failed; got {:?}",
+        result.turn_kind
+    );
+    assert!(
+        !result.reducer_used,
+        "reducer must not be invoked when no worker output is available"
+    );
+    assert_eq!(result.reducer_attempts, 0, "no reducer attempts expected");
+    assert_eq!(
+        result.worker_summaries.len(),
+        3,
+        "all three workers should appear in worker_summaries"
+    );
+    assert!(
+        result.worker_summaries.iter().all(|w| !w.succeeded),
+        "every worker should be marked succeeded=false"
+    );
+
+    // The actual contract bug: the response body must be distinguishable
+    // from a successful chat completion. Clients should not need to
+    // parse `choices[0].message.content` to discover that the call
+    // failed.
+    let body = &result.response_body;
+    let object = body.get("object").and_then(|v| v.as_str());
+
+    // Either the top-level shape is not a chat.completion, OR there is
+    // an explicit top-level `error` field. Both are acceptable; what is
+    // NOT acceptable is a body that looks exactly like a successful
+    // completion with the error text smuggled into `content`.
+    let looks_like_success = object == Some("chat.completion");
+    let has_top_level_error = body.get("error").is_some();
+    let finish_reason = body
+        .pointer("/choices/0/finish_reason")
+        .and_then(|v| v.as_str());
+    let finish_reason_signals_error =
+        finish_reason == Some("error") || finish_reason == Some("moa_failed");
+
+    assert!(
+        !looks_like_success || has_top_level_error || finish_reason_signals_error,
+        "all-workers-fail body must be distinguishable from a successful chat.completion. \
+         Got: object={object:?}, finish_reason={finish_reason:?}, has top-level error={has_top_level_error}, \
+         full body: {body}"
+    );
+}