yangzhuxinyzxandClaude Opus 5.5
899e9ed331
Runs DeepSeek-V4.1-Flash-1017 (GSQ-RCO IQ2_XXS GGUF + Engram IQ4_NL + DSpark5 draft) on 4x V100-32G (TP4, one x16 PCIe uplink, 62 GB host RAM) with expert offloading. All new paths are lossless and switchable. Prefill (G1-G7): - G1 activation memory: per-tile MoE scratch + FP32 accumulator, in-place expand_hc / large all-reduce, windowed hc_transition, pynccl all_gather, empty_cache before big passes; chunks up to 16K. - G2 cold-expert staging ring issued ahead across tiles and layers. - G3 routed down projection split by output rows (no duplicated TP boundary blocks, -2.25 GiB/GPU) with an IPC peer gather for decode. - G5 Engram readers, adaptive indexer tile, persistent activation grid. - G6 routed-only staging for short prompts. - G7 same-session prefix cache (scheduler guard + worker restore_prefix). Long context (G4): - Window layers keep 256-row private rings. - Packed KV (VLLM_V41_PACKED_KV): FP4 + E4M3 compressed latents in a shared host memfd plus a 32K GPU hot ring, FP4 index keys on GPU, CUDA decode index scores and segment-parallel exact top-k. Enables 512K. - RoPE rotates on the fly past its table; C2/Engram/guard limits and the DSpark draft length cap are lifted to the target length. Results vs phase 1: TTFT short 5.6 -> 0.8-1.1 s, 8K 27.7 -> 5.8 s, 15K 44.0 -> 11.1 s; 128K 110 s and 500K 651 s (needle passes); follow-up turns hit the prefix cache in ~1-1.7 s; fresh-text decode 57.8 -> 53.5 ms/round. Tokens at <=16K are identical with packed KV. Tooling: benchmarks/deepseek_v41/ gains the v41k SM70 extension source and build script, serve configs (phase 2: 16K, phase 3: 512K + prefix cache + tools + thinking), launch/validation tools, GPU micro-tests and HANDOFF.md. V4.1 switches are registered in vllm/envs.py and read through vllm.envs. Signed-off-by: yangzhuxinyzx <153831768+yangzhuxinyzx@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
195 lines
5.5 KiB
TOML
195 lines
5.5 KiB
TOML
[build-system]
|
|
# Should be mirrored in requirements/build/cuda.txt
|
|
requires = [
|
|
"cmake>=3.26.1",
|
|
"ninja",
|
|
"packaging>=24.2",
|
|
"patchelf>=0.19.1.0",
|
|
"setuptools>=77.0.3,<81.0.0",
|
|
"setuptools-scm>=8.0",
|
|
"setuptools-rust>=1.9.0",
|
|
"torch == 2.10.0",
|
|
"wheel",
|
|
"jinja2",
|
|
]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "1cat-vllm"
|
|
authors = [{name = "1Cat AI"}, {name = "vLLM Team"}]
|
|
license = "Apache-2.0"
|
|
license-files = ["LICENSE"]
|
|
readme = "README.md"
|
|
description = "A high-throughput and memory-efficient inference and serving engine for LLMs"
|
|
classifiers = [
|
|
"Programming Language :: Python :: 3.10",
|
|
"Programming Language :: Python :: 3.11",
|
|
"Programming Language :: Python :: 3.12",
|
|
"Programming Language :: Python :: 3.13",
|
|
"Programming Language :: Python :: 3.14",
|
|
"Intended Audience :: Developers",
|
|
"Intended Audience :: Information Technology",
|
|
"Intended Audience :: Science/Research",
|
|
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
]
|
|
requires-python = ">=3.10,<3.15"
|
|
dynamic = [ "version", "dependencies", "optional-dependencies"]
|
|
|
|
[project.urls]
|
|
Homepage="https://github.com/1CatAI/1Cat-vLLM"
|
|
Documentation="https://github.com/1CatAI/1Cat-vLLM"
|
|
|
|
[project.scripts]
|
|
vllm = "vllm.entrypoints.cli.main:main"
|
|
|
|
[project.entry-points."vllm.general_plugins"]
|
|
lora_filesystem_resolver = "vllm.plugins.lora_resolvers.filesystem_resolver:register_filesystem_resolver"
|
|
lora_hf_hub_resolver = "vllm.plugins.lora_resolvers.hf_hub_resolver:register_hf_hub_resolver"
|
|
|
|
[tool.setuptools_scm]
|
|
# no extra settings needed, presence enables setuptools-scm
|
|
|
|
[tool.setuptools.packages.find]
|
|
where = [".", "flash-attention-v100"]
|
|
include = ["vllm*", "flash_qla*", "flash_attn_v100*"]
|
|
|
|
[tool.ruff.lint.per-file-ignores]
|
|
"flash_qla/**" = ["ALL"]
|
|
"vllm/third_party/**" = ["ALL"]
|
|
"vllm/version.py" = ["F401"]
|
|
"vllm/_version.py" = ["ALL"]
|
|
# DS-V4.1 bring-up scripts (run by hand on the 4x V100 host).
|
|
"benchmarks/deepseek_v41/**" = ["E501", "E402", "E741", "B023", "UP031", "SIM115"]
|
|
|
|
[tool.ruff.lint]
|
|
select = [
|
|
# pycodestyle
|
|
"E",
|
|
# Pyflakes
|
|
"F",
|
|
# pyupgrade
|
|
"UP",
|
|
# flake8-bugbear
|
|
"B",
|
|
# flake8-implicit-str-concat
|
|
"ISC",
|
|
# flake8-simplify
|
|
"SIM",
|
|
# isort
|
|
"I",
|
|
# flake8-logging-format
|
|
"G",
|
|
]
|
|
ignore = [
|
|
# star imports
|
|
"F405", "F403",
|
|
# lambda expression assignment
|
|
"E731",
|
|
# zip without `strict=`
|
|
"B905",
|
|
# Loop control variable not used within loop body
|
|
"B007",
|
|
# f-string format
|
|
"UP032",
|
|
]
|
|
|
|
[tool.ruff.format]
|
|
docstring-code-format = true
|
|
|
|
[tool.mypy]
|
|
plugins = ['pydantic.mypy']
|
|
ignore_missing_imports = true
|
|
check_untyped_defs = true
|
|
follow_imports = "silent"
|
|
|
|
[tool.pytest.ini_options]
|
|
markers = [
|
|
"slow_test",
|
|
"skip_global_cleanup",
|
|
"core_model: enable this model test in each PR instead of only nightly",
|
|
"hybrid_model: models that contain mamba layers (including pure SSM and hybrid architectures)",
|
|
"cpu_model: enable this model test in CPU tests",
|
|
"cpu_test: mark test as CPU-only test",
|
|
"split: run this test as part of a split",
|
|
"distributed: run this test only in distributed GPU tests",
|
|
"optional: optional tests that are automatically skipped, include --optional to run them",
|
|
]
|
|
|
|
[tool.ty.src]
|
|
respect-ignore-files = true
|
|
|
|
[tool.ty.environment]
|
|
python = "./.venv"
|
|
|
|
[tool.typos.files]
|
|
# these files may be written in non english words
|
|
extend-exclude = ["flash_qla/*", "tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizers_/*",
|
|
"benchmarks/sonnet.txt", "tests/lora/data/*", "build/*",
|
|
"examples/pooling/token_embed/*", "tests/models/language/pooling/*",
|
|
"vllm/third_party/*", "vllm/entrypoints/serve/instrumentator/static/*",
|
|
"tests/entrypoints/speech_to_text/transcription/test_transcription_validation.py",
|
|
"docs/governance/process.md", "docs/assets/contributing/vllm_bench_serve_timeline.html",
|
|
"tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*", "csrc/cpu/sgl-kernels/*",
|
|
"rust/src/chat/src/renderer/deepseek_v32/fixtures/*",
|
|
"rust/src/tool-parser/src/gemma4.rs", "rust/src/text/src/output/decoded.rs",
|
|
"rust/src/tokenizer/src/incremental.rs", "rust/src/reasoning-parser/src/tests.rs"]
|
|
ignore-hidden = false
|
|
|
|
[tool.typos.default]
|
|
extend-ignore-identifiers-re = [".*[Uu][Ee][0-9][Mm][0-9].*"]
|
|
|
|
[tool.typos.default.extend-identifiers]
|
|
bbc5b7ede = "bbc5b7ede"
|
|
NOOPs = "NOOPs"
|
|
nin_shortcut = "nin_shortcut"
|
|
cudaDevAttrMaxSharedMemoryPerBlockOptin = "cudaDevAttrMaxSharedMemoryPerBlockOptin"
|
|
sharedMemPerBlockOptin = "sharedMemPerBlockOptin"
|
|
|
|
depthwise_seperable_out_channel = "depthwise_seperable_out_channel"
|
|
pard_token = "pard_token"
|
|
ptd_token_id = "ptd_token_id"
|
|
ser_de = "ser_de"
|
|
shared_memory_per_block_optin = "shared_memory_per_block_optin"
|
|
FoPE = "FoPE"
|
|
k_ot = "k_ot"
|
|
view_seperator = "view_seperator"
|
|
inverse_std_variences = "inverse_std_variences"
|
|
|
|
[tool.typos.default.extend-words]
|
|
iy = "iy"
|
|
indx = "indx"
|
|
# intel cpu features
|
|
tme = "tme"
|
|
dout = "dout"
|
|
Pn = "Pn"
|
|
arange = "arange"
|
|
thw = "thw"
|
|
subtile = "subtile"
|
|
subtiles = "subtiles"
|
|
nothink = "nothink"
|
|
HSA = "HSA"
|
|
setp = "setp"
|
|
CPY = "CPY"
|
|
thr = "thr"
|
|
Thr = "Thr"
|
|
PARD = "PARD"
|
|
pard = "pard"
|
|
AKS = "AKS"
|
|
ba = "ba"
|
|
fo = "fo"
|
|
nd = "nd"
|
|
eles = "eles"
|
|
datas = "datas"
|
|
ser = "ser"
|
|
ure = "ure"
|
|
VALU = "VALU"
|
|
# Walsh-Hadamard Transform
|
|
wht = "wht"
|
|
WHT = "WHT"
|
|
# Huawei Compute Architecture for Neural Networks
|
|
CANN = "CANN"
|
|
|
|
[tool.uv]
|
|
no-build-isolation-package = ["torch"]
|