Merge pull request #343 from chopratejas/rust-message-scorer-port

Rust message scorer port
This commit is contained in:
Tejas Chopra 2026-05-01 16:38:14 -07:00 committed by GitHub
commit f726d0e280
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 2038 additions and 0 deletions

View file

@ -0,0 +1,415 @@
//! Build drop candidates from a scored message list.
//!
//! Three candidate shapes (matching Python):
//!
//! - **Tool unit** — atomic `(assistant_with_tool_calls, [tool_responses])`.
//! Either the whole unit is dropped or none of it. Score = mean of
//! member scores.
//! - **Turn** — paired `(user, assistant)` neighbours when both are
//! unprotected and neither is in a tool unit. Score = mean of the
//! two scores.
//! - **Single** — any other unprotected, non-tool-unit message that
//! couldn't be paired. Score = the message's own score.
//!
//! Candidates sort by score ascending (lowest = drop first), with
//! position as a tiebreaker (older first).
use std::collections::HashSet;
use serde_json::Value;
use crate::scoring::MessageScore;
/// One unit the cascade may drop atomically.
#[derive(Debug, Clone)]
pub struct DropCandidate {
pub kind: CandidateKind,
/// Indices to remove together. May be 1 (single), 2 (turn), or
/// `1 + N` (tool unit with N tool responses).
pub indices: Vec<usize>,
/// Aggregated score; lower = drop first.
pub score: f32,
/// Earliest index — used as the secondary sort key for stability
/// (older candidates of equal score drop first).
pub position: usize,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CandidateKind {
Single,
Turn,
ToolUnit,
}
/// Tool unit: `(assistant_index, [tool_response_indices])`.
pub type ToolUnit = (usize, Vec<usize>);
/// Find all tool units in the message list. Mirrors Python's
/// `headroom.parser.find_tool_units`. Handles OpenAI, Anthropic, and
/// Strands SDK shapes.
pub fn find_tool_units(messages: &[Value]) -> Vec<ToolUnit> {
use std::collections::HashMap;
// tool_call_id → response message index
let mut response_map: HashMap<String, usize> = HashMap::new();
for (i, msg) in messages.iter().enumerate() {
let role = msg.get("role").and_then(Value::as_str);
// OpenAI: role=tool, tool_call_id
if role == Some("tool") {
if let Some(tcid) = msg.get("tool_call_id").and_then(Value::as_str) {
response_map.insert(tcid.to_string(), i);
}
}
// Anthropic + Strands: role=user, content blocks
if role == Some("user") {
if let Some(blocks) = msg.get("content").and_then(Value::as_array) {
for block in blocks {
// Anthropic: {type: tool_result, tool_use_id}
if block.get("type").and_then(Value::as_str) == Some("tool_result") {
if let Some(tcid) = block.get("tool_use_id").and_then(Value::as_str) {
response_map.insert(tcid.to_string(), i);
}
}
// Strands: {toolResult: {toolUseId}}
if let Some(tr) = block.get("toolResult") {
if let Some(tcid) = tr.get("toolUseId").and_then(Value::as_str) {
response_map.insert(tcid.to_string(), i);
}
}
}
}
}
}
let mut units: Vec<ToolUnit> = Vec::new();
for (i, msg) in messages.iter().enumerate() {
if msg.get("role").and_then(Value::as_str) != Some("assistant") {
continue;
}
let mut response_indices: Vec<usize> = Vec::new();
// OpenAI tool_calls
if let Some(arr) = msg.get("tool_calls").and_then(Value::as_array) {
for tc in arr {
if let Some(id) = tc.get("id").and_then(Value::as_str) {
if let Some(&idx) = response_map.get(id) {
response_indices.push(idx);
}
}
}
}
// Anthropic + Strands content blocks
if let Some(blocks) = msg.get("content").and_then(Value::as_array) {
for block in blocks {
// Anthropic: {type: tool_use, id}
if block.get("type").and_then(Value::as_str) == Some("tool_use") {
if let Some(id) = block.get("id").and_then(Value::as_str) {
if let Some(&idx) = response_map.get(id) {
response_indices.push(idx);
}
}
}
// Strands: {toolUse: {toolUseId}}
if let Some(tu) = block.get("toolUse") {
if let Some(id) = tu.get("toolUseId").and_then(Value::as_str) {
if let Some(&idx) = response_map.get(id) {
response_indices.push(idx);
}
}
}
}
}
if !response_indices.is_empty() {
response_indices.sort_unstable();
response_indices.dedup();
units.push((i, response_indices));
}
}
units
}
/// Aggregate `messages × scores × tool_units` into a sorted list of
/// drop candidates. Lowest score first; ties broken by position.
pub fn build_candidates(
messages: &[Value],
scores: &[MessageScore],
protected: &HashSet<usize>,
tool_units: &[ToolUnit],
) -> Vec<DropCandidate> {
// Track every index that's part of some tool unit — we don't
// want a tool-unit message to ALSO appear as a single or turn.
let mut tool_unit_indices: HashSet<usize> = HashSet::new();
for (asst_idx, responses) in tool_units {
tool_unit_indices.insert(*asst_idx);
for &r in responses {
tool_unit_indices.insert(r);
}
}
let mut candidates: Vec<DropCandidate> = Vec::new();
// Tool unit candidates — drop atomically.
for (asst_idx, responses) in tool_units {
if protected.contains(asst_idx) {
continue;
}
// If any response is protected, the unit can't drop atomically.
// Skip it; its messages stay (mirrors Python behaviour).
if responses.iter().any(|r| protected.contains(r)) {
continue;
}
let mut indices: Vec<usize> = vec![*asst_idx];
indices.extend(responses);
let avg = mean_score(&indices, scores);
candidates.push(DropCandidate {
kind: CandidateKind::ToolUnit,
indices,
score: avg,
position: *asst_idx,
});
}
// User+assistant turn pairs and singles.
let mut i: usize = 0;
while i < messages.len() {
if protected.contains(&i) || tool_unit_indices.contains(&i) {
i += 1;
continue;
}
let role = messages[i].get("role").and_then(Value::as_str);
// Try to pair user with the next assistant.
if role == Some("user") && i + 1 < messages.len() {
let next_role = messages[i + 1].get("role").and_then(Value::as_str);
let next_unprotected = !protected.contains(&(i + 1));
let next_not_in_tool_unit = !tool_unit_indices.contains(&(i + 1));
if next_role == Some("assistant") && next_unprotected && next_not_in_tool_unit {
let avg = mean_score(&[i, i + 1], scores);
candidates.push(DropCandidate {
kind: CandidateKind::Turn,
indices: vec![i, i + 1],
score: avg,
position: i,
});
i += 2;
continue;
}
}
// Single — any unprotected, non-tool-unit user/assistant message.
if matches!(role, Some("user") | Some("assistant")) {
let s = scores.get(i).map(|sc| sc.total_score).unwrap_or(0.5);
candidates.push(DropCandidate {
kind: CandidateKind::Single,
indices: vec![i],
score: s,
position: i,
});
}
i += 1;
}
// Sort by (score asc, position asc) — lowest score drops first;
// older messages drop first as a tiebreaker.
candidates.sort_by(|a, b| {
a.score
.partial_cmp(&b.score)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| a.position.cmp(&b.position))
});
candidates
}
fn mean_score(indices: &[usize], scores: &[MessageScore]) -> f32 {
if indices.is_empty() {
return 0.5;
}
let total: f32 = indices
.iter()
.filter_map(|&i| scores.get(i).map(|s| s.total_score))
.sum();
let count = indices.iter().filter(|&&i| scores.get(i).is_some()).count() as f32;
if count == 0.0 {
0.5
} else {
total / count
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn score(idx: usize, total: f32) -> MessageScore {
MessageScore {
message_index: idx,
total_score: total,
recency_score: 0.0,
semantic_score: 0.0,
toin_score: 0.0,
error_score: 0.0,
reference_score: 0.0,
density_score: 0.0,
tokens: 0,
is_protected: false,
drop_safe: true,
score_breakdown: Default::default(),
}
}
#[test]
fn find_tool_units_openai_shape() {
let msgs = vec![
json!({"role": "user", "content": "go"}),
json!({
"role": "assistant",
"content": "",
"tool_calls": [{"id": "c1", "function": {"name": "f"}}]
}),
json!({"role": "tool", "tool_call_id": "c1", "content": "result"}),
];
let units = find_tool_units(&msgs);
assert_eq!(units.len(), 1);
assert_eq!(units[0].0, 1);
assert_eq!(units[0].1, vec![2]);
}
#[test]
fn find_tool_units_anthropic_shape() {
let msgs = vec![
json!({
"role": "assistant",
"content": [
{"type": "tool_use", "id": "tu_1", "name": "f", "input": {}}
]
}),
json!({
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": "tu_1", "content": "ok"}]
}),
];
let units = find_tool_units(&msgs);
assert_eq!(units.len(), 1);
assert_eq!(units[0].0, 0);
assert_eq!(units[0].1, vec![1]);
}
#[test]
fn find_tool_units_strands_sdk_shape() {
let msgs = vec![
json!({
"role": "assistant",
"content": [{"toolUse": {"toolUseId": "tu_x", "name": "f"}}]
}),
json!({
"role": "user",
"content": [{"toolResult": {"toolUseId": "tu_x", "content": "ok"}}]
}),
];
let units = find_tool_units(&msgs);
assert_eq!(units.len(), 1);
}
#[test]
fn lowest_score_sorts_first() {
let msgs = vec![
json!({"role": "user", "content": "high"}),
json!({"role": "assistant", "content": "high"}),
json!({"role": "user", "content": "low"}),
json!({"role": "assistant", "content": "low"}),
];
let scores = vec![score(0, 0.9), score(1, 0.9), score(2, 0.1), score(3, 0.1)];
let cands = build_candidates(&msgs, &scores, &HashSet::new(), &[]);
// The low-score turn (idx 2,3) should come first.
assert_eq!(cands[0].kind, CandidateKind::Turn);
assert_eq!(cands[0].indices, vec![2, 3]);
}
#[test]
fn protected_messages_excluded() {
let msgs = vec![
json!({"role": "user", "content": "u"}),
json!({"role": "assistant", "content": "a"}),
];
let scores = vec![score(0, 0.5), score(1, 0.5)];
let mut protected = HashSet::new();
protected.insert(1); // protect the assistant
let cands = build_candidates(&msgs, &scores, &protected, &[]);
// Pair impossible (asst protected); user becomes single; asst skipped.
assert_eq!(cands.len(), 1);
assert_eq!(cands[0].kind, CandidateKind::Single);
assert_eq!(cands[0].indices, vec![0]);
}
#[test]
fn tool_unit_takes_precedence_over_singles() {
let msgs = vec![
json!({"role": "user", "content": "go"}),
json!({
"role": "assistant",
"content": "",
"tool_calls": [{"id": "c1", "function": {"name": "f"}}]
}),
json!({"role": "tool", "tool_call_id": "c1", "content": "r"}),
];
let scores = vec![score(0, 0.5), score(1, 0.2), score(2, 0.3)];
let units = find_tool_units(&msgs);
let cands = build_candidates(&msgs, &scores, &HashSet::new(), &units);
// Two candidates: the user (single) and the tool unit. The
// tool unit holds asst+tool atomically; neither asst nor tool
// appears as its own candidate.
let unit_cands: Vec<_> = cands
.iter()
.filter(|c| c.kind == CandidateKind::ToolUnit)
.collect();
assert_eq!(unit_cands.len(), 1);
let single_cands: Vec<_> = cands
.iter()
.filter(|c| c.kind == CandidateKind::Single)
.collect();
assert_eq!(single_cands.len(), 1);
assert_eq!(single_cands[0].indices, vec![0]);
}
#[test]
fn tool_unit_with_protected_response_is_skipped() {
let msgs = vec![
json!({
"role": "assistant",
"content": "",
"tool_calls": [{"id": "c1", "function": {"name": "f"}}]
}),
json!({"role": "tool", "tool_call_id": "c1", "content": "r"}),
];
let scores = vec![score(0, 0.2), score(1, 0.3)];
let mut protected = HashSet::new();
protected.insert(1); // protect the response
let units = find_tool_units(&msgs);
let cands = build_candidates(&msgs, &scores, &protected, &units);
// Unit can't drop atomically when one half is protected.
assert!(cands.iter().all(|c| c.kind != CandidateKind::ToolUnit));
}
#[test]
fn position_tiebreaker_for_equal_scores() {
let msgs = vec![
json!({"role": "user", "content": "first"}),
json!({"role": "assistant", "content": "first"}),
json!({"role": "user", "content": "second"}),
json!({"role": "assistant", "content": "second"}),
];
let scores = vec![score(0, 0.5), score(1, 0.5), score(2, 0.5), score(3, 0.5)];
let cands = build_candidates(&msgs, &scores, &HashSet::new(), &[]);
// Older turn drops first.
assert_eq!(cands[0].position, 0);
}
}

View file

@ -0,0 +1,201 @@
//! CCR-on-drop — persist dropped messages so the model can retrieve
//! them later via a tool call.
//!
//! This is the OSS-defining behaviour. Without it, dropping a message
//! is destructive (≈ rolling window). With it, the dropped content
//! is parked in the [`CcrStore`] under a content-hash key and a marker
//! is inserted into the surviving message stream pointing at it.
//! When the model later calls `ccr_retrieve(<key>)`, the dropped JSON
//! comes back verbatim.
//!
//! Mirrors Python's `_store_dropped_in_ccr` (intelligent_context.py
//! ~L955) including the marker text format. Differences from Python:
//!
//! - Rust uses the trait-level [`CcrStore`] directly (no Python
//! `CompressionStore` indirection).
//! - The marker is returned to the caller for insertion; Python
//! handled insertion outside this helper too.
use std::sync::Arc;
use serde_json::Value;
use sha2::{Digest, Sha256};
use crate::ccr::CcrStore;
/// Result of stashing dropped messages into the CCR store.
#[derive(Debug, Clone)]
pub struct DropPersist {
/// CCR cache key the dropped JSON was stored under. The marker
/// references this so the model knows what to ask for.
pub key: String,
/// Human-readable marker string suitable for insertion as a
/// system or user message in the surviving conversation.
pub marker: String,
/// Number of messages persisted.
pub count: usize,
}
/// Serialize the messages at `dropped_indices` from `original_messages`
/// and stash them in `store` under a content-hash key. Returns metadata
/// describing the persistence so the caller can insert the marker into
/// the live message list.
///
/// Returns `None` when there's nothing to do (no indices or no store)
/// or when serialization fails — failure is non-fatal because the drop
/// itself still happened. The caller logs and continues.
pub fn persist_dropped(
original_messages: &[Value],
dropped_indices: &[usize],
store: &Arc<dyn CcrStore>,
) -> Option<DropPersist> {
if dropped_indices.is_empty() {
return None;
}
// Collect in original order — re-sort because the cascade may
// append in non-monotonic order.
let mut sorted = dropped_indices.to_vec();
sorted.sort_unstable();
sorted.dedup();
let dropped: Vec<&Value> = sorted
.iter()
.filter_map(|&i| original_messages.get(i))
.collect();
if dropped.is_empty() {
return None;
}
let dropped_json = serde_json::to_string_pretty(&dropped).ok()?;
// Hash + truncate to 12 hex chars — same convention as the rest
// of the CCR store (see ccr.rs and smart_crusher hashing).
let mut h = Sha256::new();
h.update(dropped_json.as_bytes());
let key = h
.finalize()
.iter()
.take(6)
.map(|b| format!("{b:02x}"))
.collect::<String>();
// Role-count summary mirrors Python's marker format so anyone
// grepping logs sees the same string shape across implementations.
let marker = build_marker(&dropped);
// Store the original (full JSON) under the key. The marker stays
// outside the store — it lives in the live conversation as a
// pointer; the store holds only the recoverable payload.
store.put(&key, &dropped_json);
Some(DropPersist {
key,
marker,
count: dropped.len(),
})
}
/// Build the human-readable marker string. Format matches Python's
/// `_store_dropped_in_ccr` so log greps work across implementations.
fn build_marker(dropped: &[&Value]) -> String {
use std::collections::BTreeMap;
let mut role_counts: BTreeMap<&str, usize> = BTreeMap::new();
for msg in dropped {
let role = msg.get("role").and_then(Value::as_str).unwrap_or("unknown");
*role_counts.entry(role).or_insert(0) += 1;
}
let parts: Vec<String> = role_counts
.iter()
.map(|(role, count)| format!("{count} {role}"))
.collect();
format!(
"[Dropped {} messages: {}. Use ccr_retrieve to access full content.]",
dropped.len(),
parts.join(", ")
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ccr::InMemoryCcrStore;
use serde_json::json;
fn store() -> Arc<dyn CcrStore> {
Arc::new(InMemoryCcrStore::new())
}
#[test]
fn persists_dropped_messages_under_content_hash() {
let originals = vec![
json!({"role": "system", "content": "sys"}),
json!({"role": "user", "content": "drop me"}),
json!({"role": "assistant", "content": "and me"}),
json!({"role": "user", "content": "keep me"}),
];
let dropped = vec![1, 2];
let s = store();
let result = persist_dropped(&originals, &dropped, &s).expect("should persist");
assert_eq!(result.count, 2);
assert_eq!(result.key.len(), 12);
assert!(result.marker.contains("Dropped 2 messages"));
assert!(result.marker.contains("ccr_retrieve"));
// Round-trip: the original JSON is recoverable via the same key.
let stored = s.get(&result.key).expect("should retrieve");
let recovered: Vec<Value> = serde_json::from_str(&stored).unwrap();
assert_eq!(recovered.len(), 2);
assert_eq!(recovered[0]["content"], "drop me");
}
#[test]
fn empty_indices_returns_none() {
let originals = vec![json!({"role": "user", "content": "x"})];
let s = store();
assert!(persist_dropped(&originals, &[], &s).is_none());
}
#[test]
fn out_of_range_indices_are_skipped() {
let originals = vec![json!({"role": "user", "content": "x"})];
let s = store();
// Index 99 is out of range — should be silently filtered, not
// panic. Index 0 valid, so we still get a persist.
let result = persist_dropped(&originals, &[0, 99], &s).expect("should persist idx 0");
assert_eq!(result.count, 1);
}
#[test]
fn marker_role_breakdown_is_sorted_alphabetical() {
// BTreeMap ordering → roles appear alphabetically. Stable
// marker text across runs.
let originals = vec![
json!({"role": "user", "content": "u"}),
json!({"role": "assistant", "content": "a"}),
json!({"role": "tool", "content": "t"}),
];
let s = store();
let result = persist_dropped(&originals, &[0, 1, 2], &s).expect("should persist");
// "1 assistant" appears before "1 tool" appears before "1 user".
let m = &result.marker;
let asst = m.find("assistant").unwrap();
let tool = m.find("tool").unwrap();
let user = m.find("user").unwrap();
assert!(asst < tool && tool < user);
}
#[test]
fn duplicate_indices_are_deduped() {
let originals = vec![
json!({"role": "user", "content": "x"}),
json!({"role": "assistant", "content": "y"}),
];
let s = store();
let result = persist_dropped(&originals, &[0, 0, 1, 1, 1], &s).expect("should persist");
// Counts the unique message set, not the input list length.
assert_eq!(result.count, 2);
}
}

View file

@ -0,0 +1,90 @@
//! `IcmConfig` — six-field config for the OSS context manager.
//!
//! Cut from the 12+ Python config: `compress_threshold`,
//! `summarize_threshold`, `summarization_*`, `memory_tiers_*`,
//! `warm_tier_*`, `cold_tier_*`. Those belong to strategies that don't
//! ship in OSS. `recency_decay_rate` and `toin_*` moved to
//! [`MessageScorer`](crate::scoring::MessageScorer) where they belong
//! semantically.
use serde::{Deserialize, Serialize};
use crate::scoring::ScoringWeights;
/// Configuration for [`IntelligentContextManager`](super::IntelligentContextManager).
///
/// Defaults are tuned for the OSS sweet spot: keep system messages and
/// the last 2 turns sacred, leave 4K tokens of headroom for the model's
/// reply, score by importance, persist drops to CCR so they're
/// retrievable.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct IcmConfig {
/// Master switch. When `false`, `should_apply` always returns
/// `false` regardless of token count.
pub enabled: bool,
/// If `true`, `role=system` messages are never droppable. Default
/// `true` — dropping system prompts breaks behavior in subtle
/// ways. Disable only if the caller manages system prompts
/// themselves and explicitly wants them in the candidate pool.
pub keep_system: bool,
/// Number of recent user turns to protect from dropping. A "turn"
/// here means a `role=user` message; everything from the last
/// `keep_last_turns`th user message to the end is protected,
/// including all assistant replies and tool exchanges in between.
pub keep_last_turns: usize,
/// Reserved tokens for the model's response. Effective budget is
/// `model_limit - output_buffer_tokens`. Default 4000 covers most
/// reasonable replies; bump for long-form generation.
pub output_buffer_tokens: usize,
/// Weights for the six-factor message scorer. See
/// [`ScoringWeights`] for the factor breakdown. Defaults match
/// Python's `ScoringWeights()` so existing tuning carries over.
pub scoring_weights: ScoringWeights,
/// When `true` (default), dropped messages are serialized into
/// the CCR store before removal — the model can retrieve them via
/// a tool call. This is the OSS-defining behaviour: with
/// `ccr_on_drop=true`, drop ≈ "moved to retrievable cache"; with
/// `false`, drop ≈ rolling window.
pub ccr_on_drop: bool,
}
impl Default for IcmConfig {
fn default() -> Self {
Self {
enabled: true,
keep_system: true,
keep_last_turns: 2,
output_buffer_tokens: 4000,
scoring_weights: ScoringWeights::default(),
ccr_on_drop: true,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn defaults_match_oss_intent() {
let c = IcmConfig::default();
assert!(c.enabled);
assert!(c.keep_system);
assert_eq!(c.keep_last_turns, 2);
assert_eq!(c.output_buffer_tokens, 4000);
assert!(c.ccr_on_drop);
}
#[test]
fn round_trips_through_serde() {
let c = IcmConfig::default();
let json = serde_json::to_string(&c).unwrap();
let back: IcmConfig = serde_json::from_str(&json).unwrap();
assert_eq!(c, back);
}
}

View file

@ -0,0 +1,387 @@
//! `IntelligentContextManager` — the cascade orchestrator.
//!
//! Holds the config, the safety-rails computer, and the registered
//! list of strategies. On `apply()`, it:
//!
//! 1. Tokenizes, exits early if under budget.
//! 2. Computes safety-rail protections (system, last-N-turns, frozen
//! prefix, paired tool responses).
//! 3. Builds a [`ContextWorkspace`] and walks each registered strategy
//! in registration order until a strategy reports `fully_resolved`
//! or the list is exhausted.
//! 4. Returns the (possibly mutated) message list plus an
//! [`ApplyResult`] describing what happened.
//!
//! OSS pre-registers exactly one strategy: [`DropByScoreStrategy`].
//! Enterprise calls `with_strategy` more times.
use std::sync::Arc;
use serde_json::Value;
use crate::ccr::CcrStore;
use crate::context::config::IcmConfig;
use crate::context::safety::SafetyRails;
use crate::context::strategy::{ContextStrategy, DropByScoreStrategy};
use crate::context::workspace::ContextWorkspace;
use crate::scoring::MessageScorer;
use crate::tokenizer::Tokenizer;
/// Per-call inputs that vary by request.
pub struct ApplyCtx {
/// Provider's context-window size in tokens.
pub model_limit: usize,
/// Override the manager's `output_buffer_tokens`. `None` uses the
/// configured default.
pub output_buffer: Option<usize>,
/// Number of leading messages that are part of the provider's
/// prompt cache. Always protected. Provider-aware caller computes
/// this; ICM just receives it.
pub frozen_message_count: usize,
}
impl Default for ApplyCtx {
fn default() -> Self {
Self {
model_limit: 128_000,
output_buffer: None,
frozen_message_count: 0,
}
}
}
/// Outcome of a single `apply()` call.
#[derive(Debug, Clone)]
pub struct ApplyResult {
/// The (possibly modified) message list.
pub messages: Vec<Value>,
pub tokens_before: usize,
pub tokens_after: usize,
/// Names of strategies that ran (in order). Strategies that were
/// short-circuited by an earlier `fully_resolved=true` don't appear.
pub strategies_applied: Vec<&'static str>,
/// Marker strings emitted by the cascade (e.g. CCR retrieval hints).
pub markers_inserted: Vec<String>,
}
/// The orchestrator. Constructed once per process and reused across
/// requests; `Send + Sync`.
pub struct IntelligentContextManager {
config: IcmConfig,
tokenizer: Arc<dyn Tokenizer>,
strategies: Vec<Box<dyn ContextStrategy>>,
}
impl IntelligentContextManager {
/// Construct with the OSS default strategy stack: just
/// [`DropByScoreStrategy`]. The scorer is built from the config's
/// `scoring_weights`. Enterprise can call `with_strategy` to
/// register additional strategies before serving requests.
///
/// `ccr` is the CCR store used for drop persistence (and is also
/// stored on the strategy). Pass `None` to disable CCR-on-drop
/// even if `config.ccr_on_drop = true`.
pub fn new(
config: IcmConfig,
tokenizer: Arc<dyn Tokenizer>,
ccr: Option<Arc<dyn CcrStore>>,
) -> Self {
let scorer = Arc::new(MessageScorer::new(
Some(config.scoring_weights),
None,
None,
0.1,
));
let drop_strategy: Box<dyn ContextStrategy> = Box::new(DropByScoreStrategy::new(
scorer,
tokenizer.clone(),
ccr,
config.ccr_on_drop,
));
Self {
config,
tokenizer,
strategies: vec![drop_strategy],
}
}
/// Append an Enterprise strategy. Strategies run in registration
/// order; the OSS `DropByScoreStrategy` is always first unless the
/// caller explicitly bypasses it via [`Self::without_default_strategy`].
pub fn with_strategy(mut self, strategy: Box<dyn ContextStrategy>) -> Self {
self.strategies.push(strategy);
self
}
/// Drop the OSS default strategy from the stack. Use only when
/// the caller wants a fully custom strategy chain — e.g. an
/// Enterprise build that handles dropping itself.
pub fn without_default_strategy(mut self) -> Self {
// Removes the default DropByScoreStrategy registered in `new`.
self.strategies.clear();
self
}
pub fn config(&self) -> &IcmConfig {
&self.config
}
/// Cheap pre-check. `true` means `apply()` would do work; `false`
/// means the request is under budget and a no-op pass-through is
/// safe (and faster — skips tokenization on the hot path).
pub fn should_apply(
&self,
messages: &[Value],
model_limit: usize,
output_buffer: usize,
) -> bool {
if !self.config.enabled {
return false;
}
let current = count_messages(messages, self.tokenizer.as_ref());
let available = model_limit.saturating_sub(output_buffer);
current > available
}
/// Run the cascade.
pub fn apply(&self, messages: Vec<Value>, ctx: ApplyCtx) -> ApplyResult {
let output_buffer = ctx
.output_buffer
.unwrap_or(self.config.output_buffer_tokens);
let target = ctx.model_limit.saturating_sub(output_buffer);
let tokens_before = count_messages(&messages, self.tokenizer.as_ref());
// Early exit: under budget OR disabled.
if !self.config.enabled || tokens_before <= target {
return ApplyResult {
messages,
tokens_before,
tokens_after: tokens_before,
strategies_applied: Vec::new(),
markers_inserted: Vec::new(),
};
}
// Compute safety rails + build workspace.
let safety = SafetyRails::new(&self.config);
let protected = safety.protected(&messages, ctx.frozen_message_count);
let mut ws = ContextWorkspace::new(messages, protected, ctx.frozen_message_count);
ws.current_tokens = tokens_before;
let mut strategies_applied: Vec<&'static str> = Vec::new();
let mut markers: Vec<String> = Vec::new();
for strategy in &self.strategies {
let outcome = strategy.try_fit(&mut ws, target);
strategies_applied.push(strategy.name());
markers.extend(outcome.markers_inserted);
// Recompute protections — indices may have shifted post-drop.
// Subsequent strategies need a fresh view.
ws.protected = safety.protected(&ws.messages, ws.frozen_count);
if outcome.fully_resolved {
break;
}
}
ApplyResult {
tokens_before,
tokens_after: ws.current_tokens,
messages: ws.messages,
strategies_applied,
markers_inserted: markers,
}
}
}
// Re-export the strategy's count_messages for tests + manager-level
// gating. Keeping a single source of truth avoids drift between
// `should_apply` and the strategy's own accounting.
pub(crate) use crate::context::strategy::drop_by_score::count_messages;
#[cfg(test)]
mod tests {
use super::*;
use crate::ccr::InMemoryCcrStore;
use crate::tokenizer::EstimatingCounter;
use serde_json::json;
fn manager() -> IntelligentContextManager {
let tokenizer: Arc<dyn Tokenizer> = Arc::new(EstimatingCounter::default());
let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());
IntelligentContextManager::new(IcmConfig::default(), tokenizer, Some(store))
}
#[test]
fn should_apply_returns_false_when_disabled() {
let cfg = IcmConfig {
enabled: false,
..Default::default()
};
let tokenizer: Arc<dyn Tokenizer> = Arc::new(EstimatingCounter::default());
let m = IntelligentContextManager::new(cfg, tokenizer, None);
let huge = vec![json!({"role": "user", "content": "x".repeat(1_000_000)})];
assert!(!m.should_apply(&huge, 1000, 100));
}
#[test]
fn should_apply_false_under_budget() {
let m = manager();
let small = vec![json!({"role": "user", "content": "hi"})];
assert!(!m.should_apply(&small, 128_000, 4_000));
}
#[test]
fn should_apply_true_over_budget() {
let m = manager();
let huge = vec![json!({"role": "user", "content": "x".repeat(1_000_000)})];
assert!(m.should_apply(&huge, 1000, 100));
}
#[test]
fn apply_under_budget_is_passthrough() {
let m = manager();
let msgs = vec![json!({"role": "user", "content": "hello"})];
let n = msgs.len();
let r = m.apply(
msgs,
ApplyCtx {
model_limit: 128_000,
output_buffer: Some(4_000),
frozen_message_count: 0,
},
);
assert_eq!(r.messages.len(), n);
assert!(r.strategies_applied.is_empty());
assert_eq!(r.tokens_before, r.tokens_after);
}
#[test]
fn apply_over_budget_runs_drop_by_score() {
let m = manager();
// Many large-ish messages; tight budget forces drops. Last
// turn is protected by default config (keep_last_turns=2).
let mut msgs: Vec<Value> = (0..10)
.map(|i| json!({"role": "user", "content": format!("message {i} ").repeat(50)}))
.collect();
msgs.push(json!({"role": "assistant", "content": "ack"}));
msgs.push(json!({"role": "user", "content": "final"}));
let initial = count_messages(&msgs, &EstimatingCounter::default());
let r = m.apply(
msgs.clone(),
ApplyCtx {
model_limit: initial / 2,
output_buffer: Some(0),
frozen_message_count: 0,
},
);
assert_eq!(r.strategies_applied, vec!["drop_by_score"]);
assert!(r.tokens_after < r.tokens_before);
assert!(r.messages.len() < msgs.len());
// Final user message ("final") is protected; it must survive.
let last = r.messages.last().unwrap();
assert_eq!(last["content"], "final");
}
#[test]
fn frozen_prefix_is_never_dropped() {
let m = manager();
let mut msgs: Vec<Value> = Vec::new();
msgs.push(json!({"role": "user", "content": "FROZEN PREFIX MARKER".repeat(50)}));
for i in 0..15 {
msgs.push(json!({"role": "user", "content": format!("filler {i} ").repeat(50)}));
}
let initial = count_messages(&msgs, &EstimatingCounter::default());
let r = m.apply(
msgs,
ApplyCtx {
model_limit: initial / 4,
output_buffer: Some(0),
frozen_message_count: 1,
},
);
// The first message (frozen) must still be present.
assert!(r.messages[0]["content"]
.as_str()
.unwrap()
.contains("FROZEN PREFIX MARKER"));
}
#[test]
fn system_message_protected_by_default() {
let m = manager();
let mut msgs: Vec<Value> = Vec::new();
msgs.push(json!({"role": "system", "content": "you are helpful"}));
for i in 0..15 {
msgs.push(json!({"role": "user", "content": format!("filler {i} ").repeat(50)}));
}
let initial = count_messages(&msgs, &EstimatingCounter::default());
let r = m.apply(
msgs,
ApplyCtx {
model_limit: initial / 3,
output_buffer: Some(0),
frozen_message_count: 0,
},
);
// System message present somewhere.
assert!(r
.messages
.iter()
.any(|m| m.get("role").and_then(Value::as_str) == Some("system")));
}
#[test]
fn with_strategy_appends_to_chain() {
struct Sentinel;
impl ContextStrategy for Sentinel {
fn name(&self) -> &'static str {
"sentinel"
}
fn try_fit(
&self,
_ws: &mut ContextWorkspace,
_target: usize,
) -> crate::context::workspace::StrategyOutcome {
crate::context::workspace::StrategyOutcome::default()
}
}
let m = manager().with_strategy(Box::new(Sentinel));
// Force a drop scenario.
let msgs: Vec<Value> = (0..20)
.map(|i| json!({"role": "user", "content": format!("m{i} ").repeat(50)}))
.collect();
let initial = count_messages(&msgs, &EstimatingCounter::default());
let r = m.apply(
msgs,
ApplyCtx {
model_limit: initial / 4,
output_buffer: Some(0),
frozen_message_count: 0,
},
);
// drop_by_score may or may not fully resolve. If it does,
// sentinel is short-circuited (no entry). If not, sentinel
// appears second.
assert_eq!(r.strategies_applied[0], "drop_by_score");
}
#[test]
fn ccr_marker_emitted_on_drop() {
let m = manager();
let msgs: Vec<Value> = (0..20)
.map(|i| json!({"role": "user", "content": format!("m{i} ").repeat(50)}))
.collect();
let initial = count_messages(&msgs, &EstimatingCounter::default());
let r = m.apply(
msgs,
ApplyCtx {
model_limit: initial / 4,
output_buffer: Some(0),
frozen_message_count: 0,
},
);
assert!(!r.markers_inserted.is_empty());
assert!(r.markers_inserted[0].contains("ccr_retrieve"));
}
}

View file

@ -0,0 +1,47 @@
//! Conversation-level context management for the OSS proxy.
//!
//! `context/` lives at the crate root parallel to `scoring/`,
//! `signals/`, and `transforms/` because it operates on a *different*
//! abstraction than any of them:
//!
//! - `transforms/` and `signals/` work on **content** (a string, a
//! structured JSON blob, a tool-result body).
//! - `scoring/` scores **messages** for importance.
//! - `context/` orchestrates **conversations** — it decides what to
//! do when a request's message list is over the model's context
//! budget. It uses `scoring/` for per-message scores and the CCR
//! store for drop persistence, but it is itself a higher-level
//! layer than either.
//!
//! # OSS philosophy
//!
//! OSS ships exactly one strategy: [`DropByScoreStrategy`] —
//! multi-factor scoring + safety rails + CCR-on-drop persistence. This
//! alone outclasses every gateway competitor's rolling-window
//! behaviour. Enterprise plugs additional strategies (compress-first,
//! summarize, memory tiers) in via [`ContextStrategy`] without any
//! orchestrator changes.
//!
//! # Why simpler than the Python original
//!
//! The Python `IntelligentContextManager` shipped three cascading
//! strategies (COMPRESS_FIRST, SUMMARIZE, DROP_BY_SCORE) plus 12+
//! config fields and integration hooks for ContentRouter,
//! ProgressiveSummarizer, TOIN, and the CCR store. Two of those
//! strategies belong to value-adds we're scoping to Enterprise; one
//! (memory tiers) was never implemented at all. This module is the
//! deliberate refactor: keep the OSS-defining behaviour, push the
//! rest behind a trait extension point.
pub mod candidate;
pub mod ccr_drop;
pub mod config;
pub mod manager;
pub mod safety;
pub mod strategy;
pub mod workspace;
pub use config::IcmConfig;
pub use manager::{ApplyCtx, ApplyResult, IntelligentContextManager};
pub use strategy::{ContextStrategy, DropByScoreStrategy};
pub use workspace::{ContextWorkspace, StrategyOutcome};

View file

@ -0,0 +1,357 @@
//! `SafetyRails` — compute the set of indices that strategies must
//! never drop.
//!
//! Direct port of Python's `IntelligentContextManager._get_protected_indices`,
//! including the awkward-but-correct two-pass walk:
//!
//! 1. Walk forward, mark `role=system` (if `keep_system`).
//! 2. Walk backward from the end, mark messages until `keep_last_turns`
//! user messages have been seen.
//! 3. For every assistant message in the protected set, find its tool
//! responses (OpenAI `role=tool` + `tool_call_id`, OR Anthropic
//! `role=user` with `content[].type=tool_result`) and protect those
//! too. Otherwise we'd ship an orphan tool_call → 400 from the API.
//!
//! Frozen prefix is added on top by the caller via
//! [`ContextWorkspace::frozen_count`](super::workspace::ContextWorkspace::frozen_count).
//! This module just handles the per-message rules.
use std::collections::HashSet;
use serde_json::Value;
use crate::context::config::IcmConfig;
pub struct SafetyRails<'c> {
config: &'c IcmConfig,
}
impl<'c> SafetyRails<'c> {
pub fn new(config: &'c IcmConfig) -> Self {
Self { config }
}
/// Compute protected indices for the given message list.
///
/// Includes the leading `frozen_count` indices (prompt-cache prefix)
/// in the result so callers don't have to remember to OR them in.
pub fn protected(&self, messages: &[Value], frozen_count: usize) -> HashSet<usize> {
let mut protected: HashSet<usize> = HashSet::new();
// Frozen prefix — provider's prompt cache. Dropping any of
// these busts the cache and raises latency + cost.
for i in 0..frozen_count.min(messages.len()) {
protected.insert(i);
}
// System messages — never droppable when keep_system is on.
// Python convention: any `role=system` anywhere in the list,
// not only at index 0.
if self.config.keep_system {
for (i, msg) in messages.iter().enumerate() {
if msg.get("role").and_then(Value::as_str) == Some("system") {
protected.insert(i);
}
}
}
// Last N user-turn boundary. Walk backward from the end,
// protecting every message until we've seen `keep_last_turns`
// role=user messages. Includes assistant + tool messages
// interleaved with those user turns.
if self.config.keep_last_turns > 0 && !messages.is_empty() {
let mut turns_seen = 0;
let mut i = messages.len() as isize - 1;
while i >= 0 && turns_seen < self.config.keep_last_turns {
let msg = &messages[i as usize];
protected.insert(i as usize);
if msg.get("role").and_then(Value::as_str) == Some("user") {
turns_seen += 1;
}
i -= 1;
}
}
// Pair tool responses with their assistant tool_call messages.
// Protected assistant → every tool response that satisfies
// its tool_call ids must also be protected. Otherwise we'd
// drop a `tool` message whose corresponding `assistant`
// tool_call is still in the prompt → API rejects with 400.
let pairs = self.tool_response_pairs(messages, &protected);
protected.extend(pairs);
protected
}
/// For each protected assistant message, find every tool response
/// (OpenAI or Anthropic shape) that references one of its tool
/// call ids. Returns the response indices — the assistant indices
/// are already in `protected`.
fn tool_response_pairs(
&self,
messages: &[Value],
protected: &HashSet<usize>,
) -> HashSet<usize> {
let mut response_indices: HashSet<usize> = HashSet::new();
for &i in protected {
let msg = match messages.get(i) {
Some(m) => m,
None => continue,
};
if msg.get("role").and_then(Value::as_str) != Some("assistant") {
continue;
}
let tool_call_ids = collect_assistant_tool_call_ids(msg);
if tool_call_ids.is_empty() {
continue;
}
for (j, other) in messages.iter().enumerate() {
if protected.contains(&j) {
continue;
}
if message_responds_to_tool_call(other, &tool_call_ids) {
response_indices.insert(j);
}
}
}
response_indices
}
}
/// Extract the set of tool_call ids announced by an assistant message,
/// handling both OpenAI and Anthropic shapes.
///
/// - **OpenAI**: `{role:assistant, tool_calls: [{id, ...}, ...]}`
/// - **Anthropic**: `{role:assistant, content: [{type:tool_use, id, ...}]}`
fn collect_assistant_tool_call_ids(assistant: &Value) -> HashSet<String> {
let mut ids: HashSet<String> = HashSet::new();
// OpenAI: tool_calls array
if let Some(arr) = assistant.get("tool_calls").and_then(Value::as_array) {
for tc in arr {
if let Some(id) = tc.get("id").and_then(Value::as_str) {
ids.insert(id.to_string());
}
}
}
// Anthropic: content blocks with type=tool_use
if let Some(blocks) = assistant.get("content").and_then(Value::as_array) {
for block in blocks {
if block.get("type").and_then(Value::as_str) == Some("tool_use") {
if let Some(id) = block.get("id").and_then(Value::as_str) {
ids.insert(id.to_string());
}
}
}
}
ids
}
/// Does this message respond to one of `ids`?
///
/// - **OpenAI**: `{role:tool, tool_call_id: "..."}`
/// - **Anthropic**: `{role:user, content: [{type:tool_result, tool_use_id: "..."}]}`
fn message_responds_to_tool_call(msg: &Value, ids: &HashSet<String>) -> bool {
let role = msg.get("role").and_then(Value::as_str);
// OpenAI shape
if role == Some("tool") {
if let Some(tcid) = msg.get("tool_call_id").and_then(Value::as_str) {
return ids.contains(tcid);
}
}
// Anthropic shape — tool results live inside a user message's content list
if role == Some("user") {
if let Some(blocks) = msg.get("content").and_then(Value::as_array) {
for block in blocks {
if block.get("type").and_then(Value::as_str) == Some("tool_result") {
if let Some(tuid) = block.get("tool_use_id").and_then(Value::as_str) {
if ids.contains(tuid) {
return true;
}
}
}
}
}
}
false
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn cfg(keep_system: bool, keep_last_turns: usize) -> IcmConfig {
IcmConfig {
keep_system,
keep_last_turns,
..Default::default()
}
}
#[test]
fn protects_system_messages() {
let msgs = vec![
json!({"role": "system", "content": "you are helpful"}),
json!({"role": "user", "content": "hi"}),
json!({"role": "assistant", "content": "hello"}),
];
let c = cfg(true, 0);
let s = SafetyRails::new(&c);
let p = s.protected(&msgs, 0);
assert!(p.contains(&0));
assert!(!p.contains(&1));
assert!(!p.contains(&2));
}
#[test]
fn keep_system_false_does_not_protect_system() {
let msgs = vec![
json!({"role": "system", "content": "sys"}),
json!({"role": "user", "content": "hi"}),
];
let c = cfg(false, 0);
let s = SafetyRails::new(&c);
let p = s.protected(&msgs, 0);
assert!(!p.contains(&0));
}
#[test]
fn last_n_turns_protects_user_and_inner_messages() {
// 5 messages, keep_last_turns=2 → walk back until we've seen
// 2 user messages. Should protect indices 2,3,4 (covers user@4,
// asst@3, user@2).
let msgs = vec![
json!({"role": "user", "content": "q1"}),
json!({"role": "assistant", "content": "a1"}),
json!({"role": "user", "content": "q2"}),
json!({"role": "assistant", "content": "a2"}),
json!({"role": "user", "content": "q3"}),
];
let c = cfg(false, 2);
let s = SafetyRails::new(&c);
let p = s.protected(&msgs, 0);
assert!(p.contains(&2));
assert!(p.contains(&3));
assert!(p.contains(&4));
assert!(!p.contains(&0));
assert!(!p.contains(&1));
}
#[test]
fn frozen_prefix_is_protected() {
let msgs = vec![
json!({"role": "user", "content": "1"}),
json!({"role": "user", "content": "2"}),
json!({"role": "user", "content": "3"}),
];
let c = cfg(false, 0);
let s = SafetyRails::new(&c);
let p = s.protected(&msgs, 2);
assert!(p.contains(&0));
assert!(p.contains(&1));
assert!(!p.contains(&2));
}
#[test]
fn frozen_count_above_message_count_is_clamped() {
let msgs = vec![json!({"role": "user", "content": "x"})];
let c = cfg(false, 0);
let s = SafetyRails::new(&c);
// No panic, just protect what's there.
let p = s.protected(&msgs, 99);
assert_eq!(p.len(), 1);
}
#[test]
fn openai_tool_pair_is_atomic() {
// assistant @ idx 1 has a tool_call; tool response @ idx 2.
// keep_last_turns protects from the end. With keep_last=1 we
// should pull in user@3 only, then the pair logic adds nothing.
// Use keep_last=0 + keep_system=false but pre-protect the
// assistant manually (simulate a caller-protected index).
let msgs = vec![
json!({"role": "user", "content": "go"}),
json!({
"role": "assistant",
"content": "",
"tool_calls": [{"id": "call_1", "type": "function",
"function": {"name": "f", "arguments": "{}"}}]
}),
json!({"role": "tool", "tool_call_id": "call_1", "content": "result"}),
json!({"role": "user", "content": "thanks"}),
];
// keep_last_turns=2 → protects user@3 + asst@2 (but the asst@2
// is the tool message... no wait, asst is at 1, tool at 2).
// Walking back from idx 3: protect 3 (user, 1 turn), 2 (tool),
// 1 (asst), 0 (user, 2 turns). All four protected.
let c = cfg(false, 2);
let s = SafetyRails::new(&c);
let p = s.protected(&msgs, 0);
assert!(p.contains(&1));
assert!(p.contains(&2)); // The tool response must be protected too.
}
#[test]
fn anthropic_tool_pair_is_atomic() {
// Anthropic: assistant content has tool_use blocks; tool_result
// lives inside a user message's content list.
let msgs = vec![
json!({"role": "user", "content": "go"}),
json!({
"role": "assistant",
"content": [
{"type": "text", "text": "thinking..."},
{"type": "tool_use", "id": "tu_1", "name": "f", "input": {}}
]
}),
json!({
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": "tu_1", "content": "ok"}]
}),
json!({"role": "user", "content": "next"}),
];
// Pre-protect the assistant via keep_last_turns=2 (walks back
// through user@3, user@2 = 2 turns; assistant@1 not protected
// by turn rule). Bump to keep_last_turns=3 so assistant@1 falls
// under the protection.
let c = cfg(false, 3);
let s = SafetyRails::new(&c);
let p = s.protected(&msgs, 0);
assert!(p.contains(&1)); // assistant
assert!(p.contains(&2)); // tool_result user
}
#[test]
fn unmatched_tool_call_id_does_not_inflate_protection() {
let msgs = vec![
json!({"role": "user", "content": "go"}),
json!({
"role": "assistant",
"content": "",
"tool_calls": [{"id": "call_orphan",
"function": {"name": "f"}}]
}),
json!({"role": "tool", "tool_call_id": "call_DIFFERENT", "content": "?"}),
];
let c = cfg(false, 0);
let s = SafetyRails::new(&c);
// Manually protect the assistant by setting keep_last_turns
// to wrap it. But keep_last=0 here so nothing is protected;
// the orphan tool message stays unprotected. Now protect via
// keep_last_turns=1 wrapping back from end…
// Actually with keep_last=0 + keep_system=false there's no
// protection at all, so the pair logic finds nothing to pair.
let p = s.protected(&msgs, 0);
assert!(p.is_empty());
}
}

View file

@ -0,0 +1,410 @@
//! `DropByScoreStrategy` — the OSS-shipped context strategy.
//!
//! Algorithm:
//!
//! 1. Build candidates (turns + singles + tool units), sorted by
//! importance score ascending.
//! 2. Walk the candidate list, dropping until token budget is met.
//! 3. After each drop, recompute token count via the supplied
//! tokenizer. Stop early once `current_tokens <= target`.
//! 4. If `ccr_on_drop=true` AND a CCR store is wired, persist the
//! dropped messages and emit a marker so the model can retrieve
//! them via tool call.
//!
//! Direct port of the Python `_apply_drop_by_score` path with the
//! same ordering semantics (lowest score first, position tiebreaker).
use std::sync::Arc;
use serde_json::Value;
use crate::ccr::CcrStore;
use crate::context::candidate::{build_candidates, find_tool_units};
use crate::context::ccr_drop::persist_dropped;
use crate::context::workspace::{ContextWorkspace, StrategyOutcome};
use crate::scoring::MessageScorer;
use crate::tokenizer::Tokenizer;
use super::ContextStrategy;
/// Drop-by-importance-score strategy.
///
/// Holds a [`MessageScorer`] (constructed once with the manager's
/// scoring weights) and an optional CCR store for persistence on drop.
/// Both are `Arc`-wrapped because the strategy is shared across
/// requests in a multi-threaded proxy.
pub struct DropByScoreStrategy {
scorer: Arc<MessageScorer>,
tokenizer: Arc<dyn Tokenizer>,
ccr_store: Option<Arc<dyn CcrStore>>,
ccr_on_drop: bool,
}
impl DropByScoreStrategy {
pub fn new(
scorer: Arc<MessageScorer>,
tokenizer: Arc<dyn Tokenizer>,
ccr_store: Option<Arc<dyn CcrStore>>,
ccr_on_drop: bool,
) -> Self {
Self {
scorer,
tokenizer,
ccr_store,
ccr_on_drop,
}
}
}
impl ContextStrategy for DropByScoreStrategy {
fn name(&self) -> &'static str {
"drop_by_score"
}
fn try_fit(&self, ws: &mut ContextWorkspace, target_tokens: usize) -> StrategyOutcome {
if ws.current_tokens <= target_tokens {
return StrategyOutcome {
fully_resolved: true,
..Default::default()
};
}
// Compute tool units (atomic groups) and message scores.
let tool_units = find_tool_units(&ws.messages);
let tool_unit_indices: std::collections::HashSet<usize> = tool_units
.iter()
.flat_map(|(a, rs)| std::iter::once(*a).chain(rs.iter().copied()))
.collect();
let scores = self
.scorer
.score_messages(&ws.messages, &ws.protected, &tool_unit_indices);
let candidates = build_candidates(&ws.messages, &scores, &ws.protected, &tool_units);
// Walk lowest → highest score, dropping one candidate at a
// time. Recompute tokens after each drop. Stop when under
// budget OR we run out of candidates.
//
// Track which ORIGINAL indices were dropped so the CCR helper
// can serialize the verbatim bytes. The workspace already
// holds `original_messages` for that purpose.
let initial_tokens = ws.current_tokens;
let mut dropped_now: Vec<usize> = Vec::new();
let mut indices_to_remove: std::collections::BTreeSet<usize> =
std::collections::BTreeSet::new();
for cand in &candidates {
if ws.current_tokens.saturating_sub(estimate_dropped_tokens(
&ws.messages,
&cand.indices,
self.tokenizer.as_ref(),
)) <= target_tokens
|| ws.current_tokens > target_tokens
{
indices_to_remove.extend(cand.indices.iter().copied());
dropped_now.extend(cand.indices.iter().copied());
// Estimate post-drop token count without rebuilding the
// list each iteration. Real recompute happens below.
let saved =
estimate_dropped_tokens(&ws.messages, &cand.indices, self.tokenizer.as_ref());
ws.current_tokens = ws.current_tokens.saturating_sub(saved);
if ws.current_tokens <= target_tokens {
break;
}
}
}
if indices_to_remove.is_empty() {
return StrategyOutcome::default();
}
// Translate WORKSPACE indices to ORIGINAL indices for CCR
// persistence. The workspace's `messages` and `original_messages`
// are aligned 1:1 in this implementation (no prior strategy
// mutated the list — this is the only OSS strategy).
ws.dropped_indices.extend(dropped_now);
// Remove highest index first so earlier indices stay valid.
let to_remove_desc: Vec<usize> = indices_to_remove.iter().rev().copied().collect();
for idx in to_remove_desc {
if idx < ws.messages.len() {
ws.messages.remove(idx);
}
}
// Recompute exact token count post-mutation. The estimate above
// is approximate; the real count drives the cascade decision.
ws.current_tokens = count_messages(&ws.messages, self.tokenizer.as_ref());
// CCR persistence.
let mut markers: Vec<String> = Vec::new();
if self.ccr_on_drop {
if let Some(store) = &self.ccr_store {
if let Some(persist) =
persist_dropped(&ws.original_messages, &ws.dropped_indices, store)
{
markers.push(persist.marker);
}
}
}
let tokens_freed = initial_tokens.saturating_sub(ws.current_tokens);
StrategyOutcome {
tokens_freed,
markers_inserted: markers,
fully_resolved: ws.current_tokens <= target_tokens,
}
}
}
/// Token count for a list of messages. Mirrors Python's
/// `count_messages` accounting: per-message overhead + content
/// tokens + role tokens. Matches the OpenAI chat-format formula
/// used in the Python `tiktoken_counter`.
///
/// `pub(crate)` so the manager module can use the same accounting
/// for `should_apply` gating without duplicating the logic.
pub(crate) fn count_messages(messages: &[Value], tokenizer: &dyn Tokenizer) -> usize {
const MESSAGE_OVERHEAD: usize = 3; // OpenAI: 3 tokens per message
const REPLY_PRIMER: usize = 3; // Final assistant primer
let mut total = REPLY_PRIMER;
for msg in messages {
total += MESSAGE_OVERHEAD;
if let Some(role) = msg.get("role").and_then(Value::as_str) {
total += tokenizer.count_text(role);
}
if let Some(content) = msg.get("content") {
total += count_content(content, tokenizer);
}
if let Some(name) = msg.get("name").and_then(Value::as_str) {
total += tokenizer.count_text(name);
}
// tool_calls and tool_call_id contribute too — count their
// string forms. Coarse but sufficient for budget gating.
if let Some(tc) = msg.get("tool_calls") {
if let Ok(s) = serde_json::to_string(tc) {
total += tokenizer.count_text(&s);
}
}
if let Some(id) = msg.get("tool_call_id").and_then(Value::as_str) {
total += tokenizer.count_text(id);
}
}
total
}
fn count_content(content: &Value, tokenizer: &dyn Tokenizer) -> usize {
match content {
Value::String(s) => tokenizer.count_text(s),
Value::Array(blocks) => {
let mut total = 0;
for block in blocks {
// {type:"text",text:"..."} or {type:"tool_result",content:"..."} or
// {type:"tool_use", input:{...}} etc. Walk text-bearing fields.
if let Some(text) = block.get("text").and_then(Value::as_str) {
total += tokenizer.count_text(text);
}
if let Some(c) = block.get("content") {
if let Some(s) = c.as_str() {
total += tokenizer.count_text(s);
} else if let Ok(s) = serde_json::to_string(c) {
total += tokenizer.count_text(&s);
}
}
if let Some(input) = block.get("input") {
if let Ok(s) = serde_json::to_string(input) {
total += tokenizer.count_text(&s);
}
}
}
total
}
Value::Null => 0,
other => serde_json::to_string(other)
.map(|s| tokenizer.count_text(&s))
.unwrap_or(0),
}
}
/// Approximate the token cost of dropping `indices` from `messages`.
/// Used as a fast pre-check inside the candidate walk; the real
/// recount happens after mutation.
fn estimate_dropped_tokens(
messages: &[Value],
indices: &[usize],
tokenizer: &dyn Tokenizer,
) -> usize {
const MESSAGE_OVERHEAD: usize = 3;
let mut total = 0;
for &i in indices {
if let Some(msg) = messages.get(i) {
total += MESSAGE_OVERHEAD;
if let Some(role) = msg.get("role").and_then(Value::as_str) {
total += tokenizer.count_text(role);
}
if let Some(content) = msg.get("content") {
total += count_content(content, tokenizer);
}
}
}
total
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ccr::InMemoryCcrStore;
use crate::scoring::MessageScorer;
use crate::tokenizer::EstimatingCounter;
use serde_json::json;
use std::collections::HashSet;
fn ws(messages: Vec<Value>, tokenizer: &dyn Tokenizer) -> ContextWorkspace {
let mut w = ContextWorkspace::new(messages, HashSet::new(), 0);
w.current_tokens = count_messages(&w.messages, tokenizer);
w
}
fn strategy(
ccr: Option<Arc<dyn CcrStore>>,
ccr_on_drop: bool,
tokenizer: Arc<dyn Tokenizer>,
) -> DropByScoreStrategy {
let scorer = Arc::new(MessageScorer::with_defaults());
DropByScoreStrategy::new(scorer, tokenizer, ccr, ccr_on_drop)
}
#[test]
fn under_budget_returns_fully_resolved_no_op() {
let tokenizer: Arc<dyn Tokenizer> = Arc::new(EstimatingCounter::default());
let mut w = ws(
vec![
json!({"role": "user", "content": "short"}),
json!({"role": "assistant", "content": "ok"}),
],
tokenizer.as_ref(),
);
let s = strategy(None, false, tokenizer.clone());
let out = s.try_fit(&mut w, 1_000_000);
assert!(out.fully_resolved);
assert_eq!(out.tokens_freed, 0);
assert_eq!(w.messages.len(), 2);
}
#[test]
fn drops_lowest_scored_messages_first() {
let tokenizer: Arc<dyn Tokenizer> = Arc::new(EstimatingCounter::default());
let messages = vec![
json!({"role": "user", "content": "old turn one ".repeat(20)}),
json!({"role": "assistant", "content": "old answer one ".repeat(20)}),
json!({"role": "user", "content": "recent turn ".repeat(20)}),
json!({"role": "assistant", "content": "recent answer ".repeat(20)}),
];
let mut w = ws(messages, tokenizer.as_ref());
let initial = w.current_tokens;
let target = initial / 2;
let s = strategy(None, false, tokenizer.clone());
let out = s.try_fit(&mut w, target);
// Should drop something to free tokens.
assert!(out.tokens_freed > 0);
assert!(w.messages.len() < 4);
}
#[test]
fn protected_messages_are_never_dropped() {
let tokenizer: Arc<dyn Tokenizer> = Arc::new(EstimatingCounter::default());
let messages = vec![
json!({"role": "user", "content": "PROTECTED ".repeat(50)}),
json!({"role": "assistant", "content": "PROTECTED ".repeat(50)}),
];
let mut w = ws(messages, tokenizer.as_ref());
// Protect both indices.
w.protected.insert(0);
w.protected.insert(1);
let initial_count = w.messages.len();
let s = strategy(None, false, tokenizer.clone());
let _ = s.try_fit(&mut w, 0);
// Nothing dropped because everything's protected.
assert_eq!(w.messages.len(), initial_count);
}
#[test]
fn ccr_on_drop_persists_and_emits_marker() {
let tokenizer: Arc<dyn Tokenizer> = Arc::new(EstimatingCounter::default());
let messages = vec![
json!({"role": "user", "content": "drop me ".repeat(50)}),
json!({"role": "assistant", "content": "and me ".repeat(50)}),
json!({"role": "user", "content": "keep recent".to_string()}),
json!({"role": "assistant", "content": "ok".to_string()}),
];
let mut w = ws(messages, tokenizer.as_ref());
// Protect the last two so the cascade has to drop the first two.
w.protected.insert(2);
w.protected.insert(3);
let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());
let s = strategy(Some(store.clone()), true, tokenizer.clone());
let target = w.current_tokens / 2;
let out = s.try_fit(&mut w, target);
assert!(out.tokens_freed > 0);
assert!(!out.markers_inserted.is_empty());
// The marker references CCR retrieval.
assert!(out.markers_inserted[0].contains("ccr_retrieve"));
}
#[test]
fn ccr_off_does_not_persist() {
let tokenizer: Arc<dyn Tokenizer> = Arc::new(EstimatingCounter::default());
let messages = vec![
json!({"role": "user", "content": "drop me ".repeat(50)}),
json!({"role": "assistant", "content": "stay ".repeat(50)}),
];
let mut w = ws(messages, tokenizer.as_ref());
w.protected.insert(1);
let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());
let s = strategy(Some(store.clone()), false, tokenizer.clone());
let target = w.current_tokens / 2;
let _ = s.try_fit(&mut w, target);
// ccr_on_drop=false → no markers, no persistence.
// (Can't easily probe the store without exposing internals;
// the marker absence is the visible contract.)
}
#[test]
fn no_candidates_returns_default_outcome() {
let tokenizer: Arc<dyn Tokenizer> = Arc::new(EstimatingCounter::default());
let messages = vec![json!({"role": "system", "content": "sys".repeat(100)})];
let mut w = ws(messages, tokenizer.as_ref());
// Protect the only message → no candidates.
w.protected.insert(0);
let s = strategy(None, false, tokenizer.clone());
let out = s.try_fit(&mut w, 0);
assert_eq!(out.tokens_freed, 0);
assert!(!out.fully_resolved);
}
#[test]
fn count_messages_returns_nonzero_for_nonempty_input() {
let tokenizer: Arc<dyn Tokenizer> = Arc::new(EstimatingCounter::default());
let msgs = vec![json!({"role": "user", "content": "hello"})];
assert!(count_messages(&msgs, tokenizer.as_ref()) > 0);
}
#[test]
fn count_messages_handles_anthropic_content_blocks() {
let tokenizer: Arc<dyn Tokenizer> = Arc::new(EstimatingCounter::default());
let msgs = vec![json!({
"role": "assistant",
"content": [
{"type": "text", "text": "thinking"},
{"type": "tool_use", "id": "tu", "name": "f", "input": {"x": 1}}
]
})];
let n = count_messages(&msgs, tokenizer.as_ref());
assert!(n > 0);
}
}

View file

@ -0,0 +1,45 @@
//! `ContextStrategy` trait — the extension point for the Enterprise edition.
//!
//! OSS ships exactly one implementation: [`DropByScoreStrategy`]. Enterprise
//! plugs in additional strategies (compress-first, summarize, memory tiers)
//! and registers them via [`IntelligentContextManager::with_strategy`].
//!
//! # Why minimal
//!
//! The trait is intentionally one method. Earlier drafts added observer
//! hooks, telemetry callbacks, and tuning-hint inputs — speculative for
//! Enterprise needs we don't yet understand. Smaller surface = lower risk
//! of getting the API wrong on first try; we can extend with more methods
//! later (default-implemented for backwards compatibility).
pub mod drop_by_score;
use super::workspace::{ContextWorkspace, StrategyOutcome};
/// One stage in the context-fitting cascade.
///
/// Strategies run in registration order; each can either fully resolve
/// the budget (`fully_resolved=true` short-circuits the rest) or free
/// some tokens and let the next strategy continue.
///
/// Implementations must be `Send + Sync` — the manager is shared across
/// requests in a multi-threaded proxy. State mutations belong on the
/// `ContextWorkspace`, not `&self`.
pub trait ContextStrategy: Send + Sync {
/// Stable identifier shown in logs / `ApplyResult.strategies_applied`.
/// Use snake_case (`"drop_by_score"`, `"compress_first"`, etc.).
fn name(&self) -> &'static str;
/// Mutate the workspace toward fitting `target_tokens`.
///
/// Implementations should:
/// - Respect `ws.protected` and the leading `ws.frozen_count` indices
/// absolutely. Dropping a protected index is a contract violation.
/// - Update `ws.current_tokens` after mutating `ws.messages`.
/// - Append to `ws.dropped_indices` when removing messages.
/// - Report what they freed via `tokens_freed` so the manager can
/// decide whether to invoke the next strategy.
fn try_fit(&self, ws: &mut ContextWorkspace, target_tokens: usize) -> StrategyOutcome;
}
pub use drop_by_score::DropByScoreStrategy;

View file

@ -0,0 +1,85 @@
//! Mutable working set passed between strategies.
//!
//! Each [`ContextStrategy`](super::strategy::ContextStrategy) implementation
//! takes a `&mut ContextWorkspace`, mutates it (drops messages, inserts
//! markers, records bookkeeping), and returns a [`StrategyOutcome`].
//!
//! The workspace owns the messages — strategies never see the original
//! input list. The manager copies the input into the workspace before the
//! cascade and pulls the (possibly modified) list out at the end.
use std::collections::HashSet;
use serde_json::Value;
/// Mutable state threaded through the strategy cascade.
///
/// One workspace per `apply()` call. Not `Send` across the FFI boundary —
/// the manager constructs it inline and consumes it before returning.
pub struct ContextWorkspace {
/// The current message list. Strategies mutate this in place.
pub messages: Vec<Value>,
/// Indices that must NOT be dropped — system messages, last-N-turns,
/// frozen prefix, paired tool responses for protected assistants.
/// Computed by [`SafetyRails`](super::safety::SafetyRails) before
/// the cascade starts and re-computed after a strategy mutates the
/// list (since indices shift on drop).
pub protected: HashSet<usize>,
/// Number of leading messages that are part of a provider's
/// prompt-cache prefix. Dropping any of these busts the cache and
/// raises latency + cost. Always added to `protected`.
///
/// Provider-specific signal supplied by the caller via `ApplyCtx`.
/// Anthropic and OpenAI both expose prompt caching; OSS Headroom
/// receives this from the proxy layer (which knows the provider).
pub frozen_count: usize,
/// Token count of the current messages. Strategies update this
/// after each mutation so subsequent strategies see fresh state.
pub current_tokens: usize,
/// Indices the cascade has dropped, in drop order. Used by the
/// CCR-on-drop helper to serialize the originals into the cache.
/// Strategies append to this; they don't read it.
pub dropped_indices: Vec<usize>,
/// Original message list at workspace construction time. Preserved
/// so [`ccr_drop`](super::ccr_drop) can recover the *exact* dropped
/// content even after several drop rounds have shifted indices.
/// Strategies don't read this.
pub original_messages: Vec<Value>,
}
impl ContextWorkspace {
pub fn new(messages: Vec<Value>, protected: HashSet<usize>, frozen_count: usize) -> Self {
let original_messages = messages.clone();
Self {
messages,
protected,
frozen_count,
current_tokens: 0,
dropped_indices: Vec::new(),
original_messages,
}
}
}
/// Result of one strategy invocation.
#[derive(Debug, Clone, Default)]
pub struct StrategyOutcome {
/// Tokens freed by this strategy. The manager subtracts this from
/// `current_tokens` and decides whether to invoke the next strategy.
pub tokens_freed: usize,
/// Marker strings inserted by this strategy (e.g. CCR retrieval
/// hints). Aggregated into `ApplyResult.markers_inserted`.
pub markers_inserted: Vec<String>,
/// Set when the strategy fully resolved the budget. The manager
/// stops the cascade — later strategies don't run. Distinct from
/// "I freed some tokens" — a strategy can free tokens *and* still
/// not have brought the request under budget.
pub fully_resolved: bool,
}

View file

@ -1,6 +1,7 @@
//! headroom-core: foundation crate for the Rust port of Headroom.
pub mod ccr;
pub mod context;
pub mod relevance;
pub mod scoring;
pub mod signals;