[Bugfix][Core] Cache output blocks of reasoning requests (#1338)

#1297 stopped caching the output blocks of requests that start in reasoning
mode, assuming chat templates drop past reasoning. Qwen3.8 renders past
`reasoning` back verbatim by default, and keeps it inside a tool-call loop
even with preserve_thinking=false, so a client that returns it sends the
previous output back as the next prompt prefix.

The rule only applied in KVCacheManager.allocate_slots; the async
scheduler, used by the Qwen3.8 release profile, caches every finalized
token anyway. Remove it so both schedulers cache finalized output blocks.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
yangzhuxinyzxandClaude Opus 5.5 authored and GitHub committed 2026-11-05 21:27:27 +08:00
1 parent 96564ce015
commit d2d8e84b73
3 files changed
+2 -15

No files matched your search

+1 -8
View File
@@ -71,7 +71,7 @@ def test_eviction_releases_block_references():
assert pool.get_num_free_blocks() == free_before
def _request(boundaries, reasoning_ended=None):
def _request(boundaries):
return Request(
request_id="r",
prompt_token_ids=list(range(1000)),
@@ -79,7 +79,6 @@ def _request(boundaries, reasoning_ended=None):
max_tokens=4, extra_args={"prefix_cache_boundaries": boundaries}
),
pooling_params=None,
reasoning_ended=reasoning_ended,
)
@@ -89,12 +88,6 @@ def test_request_boundaries_align_to_recurrent_chunks():
assert request.prefix_cache_boundaries == (align, 3 * align, 999 // align * align)
def test_reasoning_requests_do_not_cache_output_tokens():
assert not _request([], reasoning_ended=False).cache_output_tokens
assert _request([], reasoning_ended=True).cache_output_tokens
assert _request([]).cache_output_tokens
class _Tokenizer:
"""ChatML-like template: one token per role marker and content word."""
+1 -3
View File
@@ -739,9 +739,7 @@ class KVCacheManager:
# "finalized" tokens are cached.
num_tokens_to_cache = min(
total_computed_tokens + num_new_tokens,
request.num_tokens
if request.cache_output_tokens
else request.num_prompt_tokens,
request.num_tokens,
)
self.coordinator.cache_blocks(request, num_tokens_to_cache)
-4
View File
@@ -174,10 +174,6 @@ class Request:
)
# Partial prefix-cache entry selected by the last cache lookup.
self.partial_prefix_entry: Any = None
# A request that starts in reasoning (thinking) mode: chat templates
# drop the reasoning from later turns, so its output blocks can never
# be a prefix of another request and are not cached.
self.cache_output_tokens = reasoning_ended is not False
# Multi-modal related
self.mm_features = mm_features or []