fix: restore tiered KV cache policy, fix split validation headroom

Models >= 50GB use Q4_0 KV cache to avoid swap thrashing on unified-memory
machines. The 480B MoE split across two Apple Silicon nodes was thrashing at
1.3 tok/s with Q8_0 KV (2GB headroom) — Q4_0 restores 13.6GB headroom and
20+ tok/s.

Split validation no longer double-counts the 10% solo-load headroom on top
of the topology planner's own VRAM budget, fixing the CI test failure in
resource_planner_returns_runtime_stage_shape.

crates/mesh-llm-host-runtime/src/inference/skippy/deployment.rs

@@ -227,7 +227,7 @@ mod tests {
             lane_count: 2,
             n_batch: None,
             n_ubatch: None,
-            kv_cache: KvCachePolicy::default(),
+            kv_cache: KvCachePolicy::for_model_size(0),
             flash_attn_type: FlashAttentionType::Auto,
             projector_path: Some("/models/mmproj.gguf".to_string()),
         };

crates/mesh-llm-host-runtime/src/inference/skippy/kv_cache.rs

@@ -22,18 +22,31 @@ pub(crate) struct KvCachePolicy {
 }
 
 impl KvCachePolicy {
-    /// Default KV cache policy: Q8_0 for both K and V.
+    const LARGE_MODEL_MIN_BYTES: u64 = 50 * 1024 * 1024 * 1024;
+
+    /// Default KV cache policy, tiered by model size.
     ///
-    /// Q8_0 gives ~2× compression over f16 with <5% speed cost across all
-    /// context lengths.  This is the universal default regardless of model
-    /// size — benchmarks show no meaningful quality degradation.
+    /// Models >= 50 GB use Q4_0 K + Q4_0 V to keep KV cache small enough
+    /// that unified-memory machines don't thrash.  On a 480B MoE split
+    /// across two Apple Silicon nodes the difference between Q8_0 and Q4_0
+    /// is the difference between swap-thrashing at 1 tok/s and running at
+    /// 20+ tok/s.
     ///
-    /// Users can override via `--cache-type-k` / `--cache-type-v` if they
-    /// want f16 (maximum precision) or q4_0 (maximum compression).
-    pub(crate) fn default() -> Self {
-        Self {
-            k_type: KvCacheType::Q8_0,
-            v_type: KvCacheType::Q8_0,
+    /// Smaller models use Q8_0 K + Q8_0 V which gives ~2× compression over
+    /// f16 with negligible quality loss.
+    ///
+    /// Users can override via `--cache-type-k` / `--cache-type-v`.
+    pub(crate) fn for_model_size(model_bytes: u64) -> Self {
+        if model_bytes >= Self::LARGE_MODEL_MIN_BYTES {
+            Self {
+                k_type: KvCacheType::Q4_0,
+                v_type: KvCacheType::Q4_0,
+            }
+        } else {
+            Self {
+                k_type: KvCacheType::Q8_0,
+                v_type: KvCacheType::Q8_0,
+            }
         }
     }
 
@@ -59,11 +72,16 @@ mod tests {
     use super::*;
 
     #[test]
-    fn default_kv_cache_is_q8_0() {
-        let policy = KvCachePolicy::default();
+    fn small_model_uses_q8_0() {
+        let policy = KvCachePolicy::for_model_size(10 * 1024 * 1024 * 1024);
         assert_eq!(policy.k_type, KvCacheType::Q8_0);
         assert_eq!(policy.v_type, KvCacheType::Q8_0);
-        assert_eq!(policy.cache_type_k(), "q8_0");
-        assert_eq!(policy.cache_type_v(), "q8_0");
+    }
+
+    #[test]
+    fn large_model_uses_q4_0() {
+        let policy = KvCachePolicy::for_model_size(50 * 1024 * 1024 * 1024);
+        assert_eq!(policy.k_type, KvCacheType::Q4_0);
+        assert_eq!(policy.v_type, KvCacheType::Q4_0);
     }
 }

crates/mesh-llm-host-runtime/src/runtime/local.rs

@@ -142,6 +142,7 @@ pub(super) struct ManagedModelController {
 pub(super) struct LocalRuntimeModelStartSpec<'a> {
     pub(super) node: &'a mesh::Node,
     pub(super) model_path: &'a Path,
+    pub(super) model_bytes: u64,
     pub(super) mmproj_override: Option<&'a Path>,
     pub(super) ctx_size_override: Option<u32>,
     pub(super) pinned_gpu: Option<&'a crate::runtime::StartupPinnedGpuTarget>,
@@ -419,7 +420,7 @@ pub(super) async fn start_runtime_local_model(
         format_gb(my_vram)
     );
 
-    let kv_cache = skippy::KvCachePolicy::default();
+    let kv_cache = skippy::KvCachePolicy::for_model_size(total_model_bytes);
     let effective_cache_type_k = spec
         .cache_type_k_override
         .unwrap_or(kv_cache.cache_type_k());
@@ -563,14 +564,28 @@ pub(super) async fn start_runtime_split_model(
             .flatten()
     }
     .context("split topology planning requires GGUF metadata")?;
+    let split_kv_policy = skippy::KvCachePolicy::for_model_size(package.source_model_bytes);
     let kv_cache_quant =
         if spec.cache_type_k_override.is_some() || spec.cache_type_v_override.is_some() {
-            let k = spec.cache_type_k_override.unwrap_or("q8_0");
-            let v = spec.cache_type_v_override.unwrap_or("q8_0");
-            models::gguf::GgufKvCacheQuant::from_llama_args(k, v)
-                .unwrap_or(models::gguf::GgufKvCacheQuant::Q8_0)
+            let k = spec
+                .cache_type_k_override
+                .unwrap_or(split_kv_policy.cache_type_k());
+            let v = spec
+                .cache_type_v_override
+                .unwrap_or(split_kv_policy.cache_type_v());
+            models::gguf::GgufKvCacheQuant::from_llama_args(k, v).unwrap_or(
+                models::gguf::GgufKvCacheQuant::from_llama_args(
+                    split_kv_policy.cache_type_k(),
+                    split_kv_policy.cache_type_v(),
+                )
+                .unwrap_or(models::gguf::GgufKvCacheQuant::Q8_0),
+            )
         } else {
-            models::gguf::GgufKvCacheQuant::Q8_0
+            models::gguf::GgufKvCacheQuant::from_llama_args(
+                split_kv_policy.cache_type_k(),
+                split_kv_policy.cache_type_v(),
+            )
+            .unwrap_or(models::gguf::GgufKvCacheQuant::Q8_0)
         };
     let kv_bytes_per_token = kv_cache_quant
         .kv_cache_bytes_per_token(&compact_meta)
@@ -881,7 +896,7 @@ async fn load_split_runtime_generation_inner(
 
     let mut ready_by_stage: HashMap<String, skippy::StageStatusSnapshot> = HashMap::new();
     let mut downstream: Option<skippy::StagePeerDescriptor> = None;
-    let kv_cache = skippy::KvCachePolicy::default();
+    let kv_cache = skippy::KvCachePolicy::for_model_size(spec.package.source_model_bytes);
     let family_policy = skippy::family_policy_for_model_path(spec.model_path, Some(spec.model_ref));
     let effective_cache_type_k = spec
         .cache_type_k_override
@@ -2531,7 +2546,7 @@ async fn start_runtime_skippy_model(
 )> {
     let port = alloc_local_port().await?;
     let context_length = plan.context_length;
-    let kv_cache = skippy::KvCachePolicy::default();
+    let kv_cache = skippy::KvCachePolicy::for_model_size(spec.model_bytes);
     let effective_cache_type_k = spec
         .cache_type_k_override
         .unwrap_or(kv_cache.cache_type_k());
@@ -2606,7 +2621,7 @@ async fn start_runtime_layer_package_model(
     tokio::sync::oneshot::Receiver<()>,
 )> {
     let context_length = plan.context_length;
-    let kv_cache = skippy::KvCachePolicy::default();
+    let kv_cache = skippy::KvCachePolicy::for_model_size(package.source_model_bytes);
     let effective_cache_type_k = spec
         .cache_type_k_override
         .unwrap_or(kv_cache.cache_type_k())

crates/mesh-llm-host-runtime/src/runtime/mod.rs

@@ -1128,6 +1128,7 @@ async fn startup_local_model_loop(params: StartupLocalModelTask) {
     let make_start_spec = || LocalRuntimeModelStartSpec {
         node: &node,
         model_path: &model_path,
+        model_bytes,
         mmproj_override: mmproj_path.as_deref(),
         ctx_size_override: ctx_size,
         pinned_gpu: pinned_gpu.as_ref(),
@@ -1433,6 +1434,7 @@ async fn startup_local_model_loop(params: StartupLocalModelTask) {
                         match start_runtime_local_model(LocalRuntimeModelStartSpec {
                             node: &node,
                             model_path: &model_path,
+                            model_bytes,
                             mmproj_override: mmproj_path.as_deref(),
                             ctx_size_override: ctx_size,
                             pinned_gpu: pinned_gpu.as_ref(),
@@ -5064,11 +5066,19 @@ async fn run_auto(
                             let requested_model = spec.clone();
                             add_serving_assignment(&node, &primary_model_name, &requested_model)
                                 .await;
+                            let runtime_model_bytes = {
+                                let p = model_path.clone();
+                                tokio::task::spawn_blocking(move || runtime_model_planning_bytes(&p))
+                                    .await
+                                    .unwrap_or(Ok(0))
+                                    .unwrap_or(0)
+                            };
                             let launch_started = Instant::now();
                             let (loaded_name, handle, death_rx) = match start_runtime_local_model(
                                 LocalRuntimeModelStartSpec {
                                     node: &node,
                                     model_path: &model_path,
+                                    model_bytes: runtime_model_bytes,
                                     mmproj_override: None,
                                     ctx_size_override: cli.ctx_size,
                                     pinned_gpu: None,

crates/mesh-llm-host-runtime/src/runtime/split_planning.rs

@@ -5,7 +5,7 @@ use skippy_coordinator::topology::{
 };
 use std::collections::HashMap;
 
-use super::local::{runtime_model_required_bytes, SplitParticipant, SplitParticipantExclusion};
+use super::local::{SplitParticipant, SplitParticipantExclusion};
 
 // VRAM budget already accounts for OS/runtime reservations (e.g. Metal's
 // recommendedMaxWorkingSetSize on macOS).  No additional headroom deduction.
@@ -360,7 +360,9 @@ pub(super) fn validate_split_capacity(
         .iter()
         .map(|participant| participant.vram_bytes)
         .sum::<u64>();
-    let required_total_bytes = runtime_model_required_bytes(package.source_model_bytes);
+    // Use raw model weight for aggregate split check — the topology planner
+    // already performed detailed per-node budgeting with KV and headroom.
+    let required_total_bytes = package.source_model_bytes;
     anyhow::ensure!(
         total_vram_bytes >= required_total_bytes,
         "{}",
@@ -382,13 +384,15 @@ pub(super) fn validate_split_capacity(
             .get(&stage.node_id)
             .copied()
             .unwrap_or_default();
-        let required_stage_bytes = runtime_model_required_bytes(stage.parameter_bytes);
+        // The topology planner already budgets VRAM including KV cache and
+        // headroom.  Do not re-apply the solo-load 10% headroom here — it
+        // double-counts and rejects topologies the planner approved.
         anyhow::ensure!(
-            node_vram >= required_stage_bytes,
+            node_vram >= stage.parameter_bytes,
             "{} assigned to {} for {model_ref} requires {}, which exceeds node capacity {}",
             stage.stage_id,
             stage.node_id.fmt_short(),
-            format_gb(required_stage_bytes),
+            format_gb(stage.parameter_bytes),
             format_gb(node_vram)
         );
     }

crates/skippy-coordinator/src/topology.rs

@@ -391,6 +391,9 @@ mod tests {
     const QWEN_CODER_480B_LAYERS: u32 = 62;
     const QWEN_CODER_480B_WEIGHT_BYTES: u64 = 315_680_000_000;
     const QWEN_CODER_480B_Q8_KV_BYTES_PER_TOKEN: u64 = 128 * 1024;
+    /// Q4_0 KV — the runtime default for models >= 50 GB to avoid memory
+    /// thrashing on unified-memory machines.
+    const QWEN_CODER_480B_Q4_KV_BYTES_PER_TOKEN: u64 = 64 * 1024;
     const LOCAL_M1_ULTRA_METAL_BYTES: u64 = 115_448_725_504;
     const STUDIO_METAL_BYTES: u64 = 239_143_780_352;
     const STUDIO_RAM_BYTES: u64 = 274_877_906_944;