diff --git a/README.md b/README.md index ada72615..59286267 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,17 @@ ______________________________________________________________________ The official `tilert==0.1.6.post2` wheel on PyPI was compiled against the following stacks. Treat these as **hard requirements**, not lower bounds (`transformers` / `tokenizers` are lower bounds). +The official `tilert==0.1.5.post2` wheel on PyPI was compiled against the following stack. Treat these as **hard requirements**, not lower bounds (`transformers` / `tokenizers` are lower bounds since v0.1.5.post2). + +| Component | Pinned version | +| ---------------- | --------------------------------------------------- | +| GPU | 8× NVIDIA **B200** | +| NVIDIA driver | Supports **CUDA 13.2** runtime | +| Operating System | Linux **x86_64**, glibc **≥ 2.28** (manylinux_2_28) | +| Python | **3.12** | +| PyTorch | **`torch==2.11.0+cu130`** | +| `transformers` | **`>= 4.46.3`** | +| `tokenizers` | **`>= 0.20.3`** | | Component | NVIDIA — DeepSeek-V3.2, GLM-5/5.1 | AMD — GLM-5.2/5.3 | | ---------------- | --------------------------------------------------- | --------------------------------------------------- | | GPU | 8× NVIDIA **B200** | 8× AMD Instinct **MI350X** / **MI355X** | @@ -118,6 +129,12 @@ docker run --rm -it --ipc=host \ ghcr.io/tile-ai/tilert-rocm-decode:0.1.6 ``` +# Inside the container — install from PyPI: +pip install tilert==0.1.5.post2 + +# Or pin the exact wheel from the GitHub Release page directly +# (same artifact, useful when PyPI is unreachable): +pip install https://github.com/tile-ai/TileRT/releases/download/v0.1.5.post2/tilert-0.1.5.post2-cp312-cp312-manylinux_2_28_x86_64.whl The ROCm image sets `HF_HUB_OFFLINE=1`; run `unset HF_HUB_OFFLINE` before downloading a checkpoint from inside the container. @@ -136,6 +153,8 @@ pip install https://github.com/tile-ai/TileRT/releases/download/v0.1.6.post2/til Verify the install: ```bash +python -c "import tilert, torch; print('tilert', tilert.__version__, '/ torch', torch.__version__, '/ cuda', torch.version.cuda)" +# Expected: tilert 0.1.5.post2 / torch 2.11.0+cu130 / cuda 13.0 python -c "import tilert, torch; print('tilert', tilert.__version__, '/ torch', torch.__version__)" # NVIDIA — expected: tilert 0.1.6.post2 / torch 2.11.0+cu130 # AMD — expected: tilert 0.1.6.post2 / torch 2.12.0+rocm7.14.0 @@ -466,6 +485,58 @@ python -m tilert.pd_vllm.pd_router --vllm-url http://:8000 \ **Note.** Running NIXL end to end (both the native and TileRT connectors in NIXL mode) lets the shared prefill use a single transfer library. Only the prefill's `--kv-transfer-config` differs from Topology A; the TileRT decode node is unchanged, and the native decode instance plus its proxy follow vLLM's usual `NixlConnector` disaggregation setup. +### Benchmarking a PD deployment(glm5.1,vllm+tilert) + +Streaming is supported since v0.1.5.post2, so `vllm bench serve` can drive the +router directly through the standard OpenAI chat backend: + +```bash +vllm bench serve \ + --backend openai-chat \ + --base-url http://:23333 \ + --model /path/to/GLM-5.1-FP8 \ + --served-model-name glm5 \ + --dataset-name random \ + --random-input-len 64000 --random-output-len 3000 \ + --random-range-ratio 0.0 \ + --num-prompts 10 --request-rate inf \ + --max-concurrency 1 --ignore-eos +``` + +#### Reference numbers: single-node co-located deployment + +One 8×B300 node running all three processes together — vLLM prefill, TileRT +decode and the router share the same 8 GPUs (the router itself is CPU-only). +GLM-5.1-FP8 with MTP, 64K input / 3K output*10, one request at a time, driven by +the `vllm bench serve` command above: + +| Latency | Mean | Median | P99 | +| ------------------------- | -------: | -------: | -------: | +| TTFT (ms) | 5153.37 | 5466.57 | 5488.29 | +| TPOT (ms) | 3.44 | 3.33 | 3.99 | +| ITL, per SSE chunk (ms) | 8.44 | 10.11 | 10.79 | +| End-to-end latency (ms) | 15471.96 | 15430.76 | 17458.84 | + +| Throughput | Value | +| --------------------------- | -------- | +| Output tokens | 193.90 tok/s | +| Input + output tokens | 4330.34 tok/s | +| Requests | 0.06 req/s | +| Duration, 10 requests | 154.72 s | + +Reading the numbers: a mean TPOT of 3.44 ms is **~291 total tokens/s & 441 output tokens/s for the +single in-flight request**, and the 5.15 s TTFT covers the full 64K-token vLLM +prefill (~12.4K tokens/s) plus the KV handoff. Aggregate output throughput +(193.9 tok/s) is lower than the per-request decode rate because prefill and +decode are serialized at concurrency 1. The mean ITL of 8.44 ms is per SSE +chunk, not per token — `8.44 / 3.44 ≈ 2.5` accepted tokens per MTP step. + +Because prefill and decode are co-located, they contend for the same SMs and +the KV handoff never leaves the host; a two-node split trades that contention +for a real RDMA hop. + + + ## Status & Future Work TileRT is currently offered as a preview release, and we’re just getting started. diff --git a/pyproject.toml b/pyproject.toml index feac4750..6bc265e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,13 @@ classifiers = [ ] dependencies = [ + # Pinned to the exact ABI the v0.1.5 wheel was built against. ``torch`` must + # come from PyTorch's cu130 index (``--index-url + # https://download.pytorch.org/whl/cu130``); installing from PyPI yields a + # CUDA build that does not match the cu130-linked tilert binary. + "torch==2.11.0", + "transformers>=4.46.3", + "tokenizers>=0.20.3", # The v0.1.6 wheel carries backends for two torch ABIs, hence the range: # the CUDA backends (DeepSeek-V3.2, GLM-5) need ``torch==2.11.0+cu130`` # from PyTorch's cu130 index (``--index-url diff --git a/requirements.txt b/requirements.txt index 9ea62fed..ef661b24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,10 @@ # # pip install -r requirements.txt # +# The recommended path remains the prebuilt Docker image (see README). +torch==2.11.0 +transformers>=4.46.3 +tokenizers>=0.20.3 # The recommended path remains the prebuilt Docker images (see README). torch>=2.11,<2.13 transformers>=4.46.3 diff --git a/tilert/pd_vllm/pd_router.py b/tilert/pd_vllm/pd_router.py index ac487945..143d9d7e 100644 --- a/tilert/pd_vllm/pd_router.py +++ b/tilert/pd_vllm/pd_router.py @@ -189,6 +189,37 @@ def _json_or_none(response) -> dict | None: return None +# Client fields that must not survive into the prefill request, which is +# forwarded verbatim apart from the fields we set: stream_options contradicts +# the stream=False we force (vLLM rejects the pair with a 400 during body +# parsing), and max_completion_tokens takes precedence over max_tokens, so it +# would override our max_tokens=1. Streaming clients send both. +_PREFILL_DROP_FIELDS = ("stream_options", "max_completion_tokens") + + +def build_prefill_body(path: str, body: dict, node: DecodeNode) -> dict: + """The vLLM request that prefills only and hands the KV state to ``node``. + + Lives outside ``build_app`` so the rewrite can be exercised without a + router process, a vLLM instance or a decode node. + """ + prefill_body = dict(body) + prefill_body["max_tokens"] = 1 + prefill_body["stream"] = False + for field in _PREFILL_DROP_FIELDS: + prefill_body.pop(field, None) + if path.endswith("chat/completions"): + prefill_body["logprobs"] = True + prefill_body["top_logprobs"] = 1 + else: + prefill_body["logprobs"] = 1 + prefill_body["kv_transfer_params"] = { + "tilert_host": node.host, + "tilert_ctrl_port": node.ctrl_port, + } + return prefill_body + + class RouterCtx: def __init__( @@ -271,6 +302,10 @@ def health(): def pool_status(): return {"nodes": [{"host": n.host, "busy": n.busy} for n in pool.nodes]} + # ── shared prefill step ────────────────────────────────────────────── + def _prefill(path, body, node): + prefill_body = build_prefill_body(path, body, node) + r = requests.post(f"{ctx.vllm_url}{path}", json=prefill_body, timeout=600) def _prefill(path, body, node, logprobs_req=None): prefill_body = build_prefill_body(path, body, node, logprobs_req, ctx.gen_defaults) r = requests.post(f"{ctx.vllm_url}{path}", json=prefill_body, timeout=_HTTP_TIMEOUT_S) @@ -284,6 +319,10 @@ def _prefill(path, body, node, logprobs_req=None): return r.json() def _sampling_of(body): + return {k: body[k] for k in ("temperature", "top_p", "top_k", "ignore_eos") if k in body} + + def _max_tokens_of(body): + return int(body.get("max_tokens") or body.get("max_completion_tokens") or 256) forwarded = ( "temperature", "top_p", @@ -627,6 +666,52 @@ async def _handle_stream(path: str, body: dict, request: Request): chunk_id = prefill["id"] model = prefill.get("model") prompt_tokens = (prefill.get("usage") or {}).get("prompt_tokens") + parser = ctx.parser(_thinking_enabled(body)) + + def _chunk(delta: dict, finish=None, usage=None) -> str: + payload = { + "id": chunk_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], + } + if usage is not None: + payload["usage"] = usage + return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" + + def _usage_chunk(usage: dict) -> str: + payload = { + "id": chunk_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [], + "usage": usage, + } + return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" + + def _event_delta(ev: dict) -> dict: + if ev["kind"] == "reasoning": + return {"reasoning_content": ev["text"]} + if ev["kind"] == "content": + return {"content": ev["text"]} + return { + "tool_calls": [ + { + "index": ev["index"], + "id": ev["id"], + "type": "function", + "function": {"name": ev["name"], "arguments": ev["arguments"]}, + } + ] + } + + def _fire_cancel(): + try: + requests.post(f"{node.http_base}/pd/cancel", json={"rid": rid}, timeout=5) + except Exception: + logger.warning("cancel POST failed for %s", rid) parser = ctx.parser(req.thinking) created = int(time.time()) _chunk = functools.partial(sse_chunk, chunk_id=chunk_id, model=model, created=created) @@ -687,6 +772,21 @@ async def _gen(): finish_reason = "stop" if client_gone: logger.info("client gone mid-stream for %s", rid) + return # finally fires the cancel + if sess is not None: + for ev in sess.finish(): + if ev["kind"] == "tool": + saw_tool = True + yield _chunk(_event_delta(ev)) + if saw_tool: + finish_reason = "tool_calls" + yield _chunk({}, finish=finish_reason) + yield _usage_chunk( + { + "prompt_tokens": prompt_tokens, + "completion_tokens": n_tokens, + "total_tokens": (prompt_tokens or 0) + n_tokens, + } return for _c in out.flush_held(): yield _c