[Bugfix][Core] Cache output blocks of reasoning requests (#1338)
#1297 stopped caching the output blocks of requests that start in reasoning mode, assuming chat templates drop past reasoning. Qwen3.8 renders past `reasoning` back verbatim by default, and keeps it inside a tool-call loop even with preserve_thinking=false, so a client that returns it sends the previous output back as the next prompt prefix. The rule only applied in KVCacheManager.allocate_slots; the async scheduler, used by the Qwen3.8 release profile, caches every finalized token anyway. Remove it so both schedulers cache finalized output blocks. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
96564ce015
commit
d2d8e84b73
3 files changed
+2
-15
No files matched your search
@@ -71,7 +71,7 @@ def test_eviction_releases_block_references():
|
||||
assert pool.get_num_free_blocks() == free_before
|
||||
|
||||
|
||||
def _request(boundaries, reasoning_ended=None):
|
||||
def _request(boundaries):
|
||||
return Request(
|
||||
request_id="r",
|
||||
prompt_token_ids=list(range(1000)),
|
||||
@@ -79,7 +79,6 @@ def _request(boundaries, reasoning_ended=None):
|
||||
max_tokens=4, extra_args={"prefix_cache_boundaries": boundaries}
|
||||
),
|
||||
pooling_params=None,
|
||||
reasoning_ended=reasoning_ended,
|
||||
)
|
||||
|
||||
|
||||
@@ -89,12 +88,6 @@ def test_request_boundaries_align_to_recurrent_chunks():
|
||||
assert request.prefix_cache_boundaries == (align, 3 * align, 999 // align * align)
|
||||
|
||||
|
||||
def test_reasoning_requests_do_not_cache_output_tokens():
|
||||
assert not _request([], reasoning_ended=False).cache_output_tokens
|
||||
assert _request([], reasoning_ended=True).cache_output_tokens
|
||||
assert _request([]).cache_output_tokens
|
||||
|
||||
|
||||
class _Tokenizer:
|
||||
"""ChatML-like template: one token per role marker and content word."""
|
||||
|
||||
|
||||
@@ -739,9 +739,7 @@ class KVCacheManager:
|
||||
# "finalized" tokens are cached.
|
||||
num_tokens_to_cache = min(
|
||||
total_computed_tokens + num_new_tokens,
|
||||
request.num_tokens
|
||||
if request.cache_output_tokens
|
||||
else request.num_prompt_tokens,
|
||||
request.num_tokens,
|
||||
)
|
||||
self.coordinator.cache_blocks(request, num_tokens_to_cache)
|
||||
|
||||
|
||||
@@ -174,10 +174,6 @@ class Request:
|
||||
)
|
||||
# Partial prefix-cache entry selected by the last cache lookup.
|
||||
self.partial_prefix_entry: Any = None
|
||||
# A request that starts in reasoning (thinking) mode: chat templates
|
||||
# drop the reasoning from later turns, so its output blocks can never
|
||||
# be a prefix of another request and are not cached.
|
||||
self.cache_output_tokens = reasoning_ended is not False
|
||||
|
||||
# Multi-modal related
|
||||
self.mm_features = mm_features or []
|
||||
|
||||
Reference in new issue
Block a user