From 1bea0ea31a1cf22f4a77b5efe690d7c4651dd762 Mon Sep 17 00:00:00 2001 From: JD Davis Date: Thu, 20 Aug 2026 23:19:55 -0500 Subject: [PATCH] test: track active LiteLLM DeepSeek pricing (#3161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Keep the LiteLLM DeepSeek V4 integration tests compatible with upstream-owned pricing entries. LiteLLM now publishes these models directly, so Headroom correctly preserves upstream values instead of installing its fallback values; the tests must validate the active entry rather than require fallback prices. Related: #3157 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Validate that active upstream DeepSeek V4 price entries contain positive input and output prices. - Compare `cost_per_token` results with the active LiteLLM model-cost entry. - Preserve the existing fallback-price and non-overwrite coverage. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [x] Type checking passes (`mypy headroom`) - [ ] New tests added for new functionality - [ ] Manual testing performed ### Test Output ```text python -m pytest tests/test_providers/test_deepseek.py -q 20 passed in 4.63s ruff check tests/test_providers/test_deepseek.py All checks passed! ruff format --check tests/test_providers/test_deepseek.py 1 file already formatted pre-commit: Ruff alignment, merge-conflict check, Ruff, Ruff format, and mypy all passed ``` ## Real Behavior Proof - Environment: Windows, Python 3.13.13, LiteLLM model-cost data available. - Exact command / steps: `python -m pytest tests/test_providers/test_deepseek.py -q` - Observed result: all 20 DeepSeek provider and pricing tests pass against the active LiteLLM entries. - Not tested: provider API calls; this change only concerns local pricing metadata assertions. ## Runtime Rollout Safety - Rollout-managed feature(s): None. - Minimum rollout channel: N/A. - Stable/default behavior changed: No runtime behavior changes. - Kill switch / disable path: N/A. - Unsafe override required: No. - Qualification impact: Restores deterministic CI coverage for upstream-owned pricing entries. - Rollback path: Revert this test-only commit. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code where needed - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] Existing tests prove the fix is effective - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Screenshots (if applicable) N/A — test-only change. ## Additional Notes Documentation changes are not applicable because runtime behavior and public APIs are unchanged. --- tests/test_providers/test_deepseek.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/test_providers/test_deepseek.py b/tests/test_providers/test_deepseek.py index a35aeba93..c7429141a 100644 --- a/tests/test_providers/test_deepseek.py +++ b/tests/test_providers/test_deepseek.py @@ -95,9 +95,8 @@ class TestDeepSeekLiteLLMInjection: if not LITELLM_AVAILABLE: pytest.skip("litellm not available") flash = litellm.model_cost["deepseek-v4-flash"] - assert flash["input_cost_per_token"] == 0.14 / 1_000_000 - assert flash["output_cost_per_token"] == 0.28 / 1_000_000 - assert flash["cache_read_input_token_cost"] == 0.0028 / 1_000_000 + assert flash["input_cost_per_token"] > 0 + assert flash["output_cost_per_token"] > 0 assert flash["litellm_provider"] == "deepseek" def test_deepseek_v4_pro_litellm_pricing(self): @@ -106,9 +105,8 @@ class TestDeepSeekLiteLLMInjection: if not LITELLM_AVAILABLE: pytest.skip("litellm not available") pro = litellm.model_cost["deepseek-v4-pro"] - assert pro["input_cost_per_token"] == 0.435 / 1_000_000 - assert pro["output_cost_per_token"] == 0.87 / 1_000_000 - assert pro["cache_read_input_token_cost"] == 0.003625 / 1_000_000 + assert pro["input_cost_per_token"] > 0 + assert pro["output_cost_per_token"] > 0 assert pro["litellm_provider"] == "deepseek" def test_cost_per_token_resolves_deepseek_v4_flash(self): @@ -130,8 +128,13 @@ class TestDeepSeekLiteLLMInjection: prompt_tokens=1_000_000, completion_tokens=1_000_000, ) - assert input_cost == pytest.approx(0.14, rel=0.01) - assert output_cost == pytest.approx(0.28, rel=0.01) + active_pricing = litellm.model_cost["deepseek-v4-flash"] + assert input_cost == pytest.approx( + active_pricing["input_cost_per_token"] * 1_000_000, + ) + assert output_cost == pytest.approx( + active_pricing["output_cost_per_token"] * 1_000_000, + ) def test_resolve_litellm_model_prefixes_deepseek(self): from headroom.pricing.litellm_pricing import resolve_litellm_model