From 290238f39854bbeffc0d8d9c93727642c3f16be1 Mon Sep 17 00:00:00 2001 From: Garm Date: Thu, 30 Apr 2026 16:56:58 +0900 Subject: [PATCH] fix(traffic-learner): raise min-evidence default and make it configurable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The traffic learner was emitting one-shot error_recovery patterns that contradicted each other and bloated MEMORY.md with low-signal noise. Two issues drove this: 1. The shutdown flush bypassed the evidence gate: the in-memory _min_evidence was set to 2, but on stop() the gate dropped to 1, so every singleton pattern got persisted at session end. This is the opposite of how evidence thresholding should work — singletons are the least trustworthy patterns, not the most. 2. The default min_evidence of 2 is too low to filter noise from the matchers, which pair up failed/successful tool calls within a small sliding window without a strong semantic check that the calls are actually related. Changes: - Raise default min_evidence from 2 to 5 in TrafficLearner. - Remove the shutdown-relaxation in flush_to_files; require self._min_evidence at all times, including on stop(). - Add traffic_learning_min_evidence to ProxyConfig (default 5). - Add --min-evidence CLI flag with HEADROOM_MIN_EVIDENCE envvar so users and embedded clients (desktop apps, plugins) can tune the threshold without source changes. - Thread the config value through HeadroomProxy into TrafficLearner. - Tests: cover default propagation and custom value flow. Co-Authored-By: Claude Opus 4.7 (1M context) --- headroom/cli/proxy.py | 13 +++++++++++++ headroom/memory/traffic_learner.py | 9 ++++----- headroom/proxy/models.py | 3 +++ headroom/proxy/server.py | 1 + tests/test_memory/test_learn_flag.py | 22 ++++++++++++++++++++++ 5 files changed, 43 insertions(+), 5 deletions(-) diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.py index ced02eb9d..8f77c6058 100644 --- a/headroom/cli/proxy.py +++ b/headroom/cli/proxy.py @@ -278,6 +278,17 @@ from .main import main is_flag=True, help="Explicitly disable traffic learning even when --memory is set.", ) +@click.option( + "--min-evidence", + type=int, + default=None, + envvar="HEADROOM_MIN_EVIDENCE", + help=( + "Minimum number of times a pattern must be observed before it is " + "persisted to memory. Higher values reduce one-shot noise at the " + "cost of slower learning. Default: 5. (env: HEADROOM_MIN_EVIDENCE)" + ), +) # Backend configuration @click.option( "--backend", @@ -380,6 +391,7 @@ def proxy( memory_qdrant_api_key: str | None, learn: bool, no_learn: bool, + min_evidence: int | None, backend: str, anyllm_provider: str, anthropic_api_url: str | None, @@ -542,6 +554,7 @@ def proxy( # Stateless mode disables learning (requires filesystem) traffic_learning_enabled=False if is_stateless else (learn and not no_learn), traffic_learning_agent_type=os.environ.get("HEADROOM_AGENT_TYPE", "unknown"), + traffic_learning_min_evidence=min_evidence if min_evidence is not None else 5, # Backend (Anthropic direct, Bedrock, LiteLLM, or any-llm) backend=backend, bedrock_region=bedrock_region or region, diff --git a/headroom/memory/traffic_learner.py b/headroom/memory/traffic_learner.py index ed7736fe5..bf9a6ac5e 100644 --- a/headroom/memory/traffic_learner.py +++ b/headroom/memory/traffic_learner.py @@ -231,7 +231,7 @@ class TrafficLearner: agent_type: str = "unknown", max_history: int = 20, dedup_window: int = 100, - min_evidence: int = 2, + min_evidence: int = 5, ) -> None: """Initialize the traffic learner. @@ -398,10 +398,9 @@ class TrafficLearner: if not patterns: return - # Evidence gate: at shutdown accept single-evidence rows; during live - # flushes require 2+ to suppress one-off noise. - min_evidence = 1 if self._stopping else 2 - patterns = [p for p in patterns if p.evidence_count >= min_evidence] + # Evidence gate: require self._min_evidence corroborations to flush, + # including at shutdown. One-shot singletons are noise, not signal. + patterns = [p for p in patterns if p.evidence_count >= self._min_evidence] if not patterns: return diff --git a/headroom/proxy/models.py b/headroom/proxy/models.py index 4bf68eff5..0f94974cb 100644 --- a/headroom/proxy/models.py +++ b/headroom/proxy/models.py @@ -203,6 +203,9 @@ class ProxyConfig: memory_inject_tools: bool = True traffic_learning_enabled: bool = False traffic_learning_agent_type: str = "unknown" # Which agent is being wrapped + # Minimum evidence count before a learned pattern is persisted to memory. + # Higher values reduce one-shot noise at the cost of slower learning. + traffic_learning_min_evidence: int = 5 memory_use_native_tool: bool = False memory_inject_context: bool = True memory_top_k: int = 10 diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py index 2d96719ba..2e62ded9e 100644 --- a/headroom/proxy/server.py +++ b/headroom/proxy/server.py @@ -534,6 +534,7 @@ class HeadroomProxy( self.traffic_learner = TrafficLearner( user_id=os.environ.get("HEADROOM_USER_ID", os.environ.get("USER", "default")), agent_type=config.traffic_learning_agent_type, + min_evidence=config.traffic_learning_min_evidence, ) # Code graph file watcher (live reindex on file changes) diff --git a/tests/test_memory/test_learn_flag.py b/tests/test_memory/test_learn_flag.py index c32d8df54..2e72bd6d5 100644 --- a/tests/test_memory/test_learn_flag.py +++ b/tests/test_memory/test_learn_flag.py @@ -102,6 +102,28 @@ class TestHeadroomProxyTrafficLearner: assert proxy.traffic_learner is not None assert proxy.traffic_learner._backend is None + def test_min_evidence_defaults_to_five(self): + """Default ProxyConfig has min_evidence=5; learner inherits it.""" + config = ProxyConfig( + memory_enabled=True, + traffic_learning_enabled=True, + ) + assert config.traffic_learning_min_evidence == 5 + proxy = HeadroomProxy(config) + assert proxy.traffic_learner is not None + assert proxy.traffic_learner._min_evidence == 5 + + def test_min_evidence_propagates_to_learner(self): + """A custom min_evidence flows from ProxyConfig into TrafficLearner.""" + config = ProxyConfig( + memory_enabled=True, + traffic_learning_enabled=True, + traffic_learning_min_evidence=10, + ) + proxy = HeadroomProxy(config) + assert proxy.traffic_learner is not None + assert proxy.traffic_learner._min_evidence == 10 + # ============================================================================= # CLI Flag Resolution Tests (simulates CLI logic without running Click)