fix(traffic-learner): raise min-evidence default and make it configurable

The traffic learner was emitting one-shot error_recovery patterns that
contradicted each other and bloated MEMORY.md with low-signal noise. Two
issues drove this:

1. The shutdown flush bypassed the evidence gate: the in-memory
   _min_evidence was set to 2, but on stop() the gate dropped to 1, so
   every singleton pattern got persisted at session end. This is the
   opposite of how evidence thresholding should work — singletons are
   the least trustworthy patterns, not the most.

2. The default min_evidence of 2 is too low to filter noise from the
   matchers, which pair up failed/successful tool calls within a small
   sliding window without a strong semantic check that the calls are
   actually related.

Changes:
- Raise default min_evidence from 2 to 5 in TrafficLearner.
- Remove the shutdown-relaxation in flush_to_files; require
  self._min_evidence at all times, including on stop().
- Add traffic_learning_min_evidence to ProxyConfig (default 5).
- Add --min-evidence CLI flag with HEADROOM_MIN_EVIDENCE envvar so
  users and embedded clients (desktop apps, plugins) can tune the
  threshold without source changes.
- Thread the config value through HeadroomProxy into TrafficLearner.
- Tests: cover default propagation and custom value flow.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garm 2026-04-30 16:56:58 +09:00
parent 0397104358
commit 290238f398
5 changed files with 43 additions and 5 deletions

View file

@ -278,6 +278,17 @@ from .main import main
is_flag=True,
help="Explicitly disable traffic learning even when --memory is set.",
)
@click.option(
"--min-evidence",
type=int,
default=None,
envvar="HEADROOM_MIN_EVIDENCE",
help=(
"Minimum number of times a pattern must be observed before it is "
"persisted to memory. Higher values reduce one-shot noise at the "
"cost of slower learning. Default: 5. (env: HEADROOM_MIN_EVIDENCE)"
),
)
# Backend configuration
@click.option(
"--backend",
@ -380,6 +391,7 @@ def proxy(
memory_qdrant_api_key: str | None,
learn: bool,
no_learn: bool,
min_evidence: int | None,
backend: str,
anyllm_provider: str,
anthropic_api_url: str | None,
@ -542,6 +554,7 @@ def proxy(
# Stateless mode disables learning (requires filesystem)
traffic_learning_enabled=False if is_stateless else (learn and not no_learn),
traffic_learning_agent_type=os.environ.get("HEADROOM_AGENT_TYPE", "unknown"),
traffic_learning_min_evidence=min_evidence if min_evidence is not None else 5,
# Backend (Anthropic direct, Bedrock, LiteLLM, or any-llm)
backend=backend,
bedrock_region=bedrock_region or region,

View file

@ -231,7 +231,7 @@ class TrafficLearner:
agent_type: str = "unknown",
max_history: int = 20,
dedup_window: int = 100,
min_evidence: int = 2,
min_evidence: int = 5,
) -> None:
"""Initialize the traffic learner.
@ -398,10 +398,9 @@ class TrafficLearner:
if not patterns:
return
# Evidence gate: at shutdown accept single-evidence rows; during live
# flushes require 2+ to suppress one-off noise.
min_evidence = 1 if self._stopping else 2
patterns = [p for p in patterns if p.evidence_count >= min_evidence]
# Evidence gate: require self._min_evidence corroborations to flush,
# including at shutdown. One-shot singletons are noise, not signal.
patterns = [p for p in patterns if p.evidence_count >= self._min_evidence]
if not patterns:
return

View file

@ -203,6 +203,9 @@ class ProxyConfig:
memory_inject_tools: bool = True
traffic_learning_enabled: bool = False
traffic_learning_agent_type: str = "unknown" # Which agent is being wrapped
# Minimum evidence count before a learned pattern is persisted to memory.
# Higher values reduce one-shot noise at the cost of slower learning.
traffic_learning_min_evidence: int = 5
memory_use_native_tool: bool = False
memory_inject_context: bool = True
memory_top_k: int = 10

View file

@ -534,6 +534,7 @@ class HeadroomProxy(
self.traffic_learner = TrafficLearner(
user_id=os.environ.get("HEADROOM_USER_ID", os.environ.get("USER", "default")),
agent_type=config.traffic_learning_agent_type,
min_evidence=config.traffic_learning_min_evidence,
)
# Code graph file watcher (live reindex on file changes)

View file

@ -102,6 +102,28 @@ class TestHeadroomProxyTrafficLearner:
assert proxy.traffic_learner is not None
assert proxy.traffic_learner._backend is None
def test_min_evidence_defaults_to_five(self):
"""Default ProxyConfig has min_evidence=5; learner inherits it."""
config = ProxyConfig(
memory_enabled=True,
traffic_learning_enabled=True,
)
assert config.traffic_learning_min_evidence == 5
proxy = HeadroomProxy(config)
assert proxy.traffic_learner is not None
assert proxy.traffic_learner._min_evidence == 5
def test_min_evidence_propagates_to_learner(self):
"""A custom min_evidence flows from ProxyConfig into TrafficLearner."""
config = ProxyConfig(
memory_enabled=True,
traffic_learning_enabled=True,
traffic_learning_min_evidence=10,
)
proxy = HeadroomProxy(config)
assert proxy.traffic_learner is not None
assert proxy.traffic_learner._min_evidence == 10
# =============================================================================
# CLI Flag Resolution Tests (simulates CLI logic without running Click)