vLLM 部署深度指南：高吞吐 LLM 推理引擎

┌─────────────────────────────────────────────┐
│           OpenAI Compatible API             │
│         (Streaming / Function Calling)      │
├─────────────────────────────────────────────┤
│          Request Scheduler                  │
│    (Continuous Batching + Priority Queue)   │
├──────────────┬──────────────────────────────┤
│  PagedAttention│   ParallelWorker           │
│  KV Cache Mgr  │   (Tensor Pipeline Parallel)│
├──────────────┴──────────────────────────────┤
│           GPU Workers (CUDA)                │
│   FlashAttention / FlashInfer Backend       │
└─────────────────────────────────────────────┘

逻辑视图 (Token Sequence):
  [BOS] [Hello] [world] [How] [are] [you] [?]
   │     │       │       │      │      │     │
物理块映射 (Block Table):       ┌──────┐
  Block 0 → GPU 0x1000  ←───── │ B0   │ [BOS Hello world How]
  Block 1 → GPU 0x2000  ←───── │ B1   │ [are you ? _ _ _ ...]
                                └──────┘

# vLLM 内部 KV Cache 管理（简化示意）
class PagedAttentionKVCache:
    def __init__(self, num_blocks, block_size=16, num_heads=32, head_size=128):
        self.block_size = block_size
        # 预分配所有 block 的显存池
        self.kv_pool = torch.empty(
            num_blocks, block_size, 2, num_heads, head_size,
            device='cuda', dtype=torch.float16
        )
        self.free_blocks = list(range(num_blocks))
        self.block_tables = {}  # seq_id → [block_indices]

    def allocate(self, seq_id, num_tokens):
        blocks_needed = (num_tokens + self.block_size - 1) // self.block_size
        self.block_tables[seq_id] = []
        for _ in range(blocks_needed):
            if not self.free_blocks:
                raise OOMError("KV Cache exhausted")
            block_idx = self.free_blocks.pop(0)
            self.block_tables[seq_id].append(block_idx)

    def append_kv(self, seq_id, keys, values):
        # 写入对应物理块，无需连续
        table = self.block_tables[seq_id]
        last_block = table[-1]
        # ... write to physical block

时间 →  T0   T1   T2   T3   T4   T5
Req A: [gen] [gen] [gen] [done] ----  ----
Req B: [gen] [gen] [gen] [gen] [gen] [done]
Req C: [gen] [gen] [done] ----  ----  ----
       ↑ GPU 在 A、C 完成后空等 B

时间 →  T0   T1   T2   T3   T4   T5
Req A: [gen] [gen] [gen] [done]
Req B: [gen] [gen] [gen] [gen] [gen] [done]
Req C: [gen] [gen] [done]
Req D:       [gen] [gen] [gen] [done]       ← A 完成后立即插入
Req E:                  [gen] [gen] [done]  ← C 完成后插入
       ↑ GPU 持续满载

# vLLM Scheduler 核心逻辑（简化）
class Scheduler:
    def __init__(self, max_num_seqs=256, max_tokens_per_batch=8192):
        self.waiting: list[SeqGroup] = []
        self.running: list[SeqGroup] = []
        self.max_num_seqs = max_num_seqs

    def schedule(self) -> ScheduledBatch:
        scheduled = []
        total_tokens = 0

        # 1. 优先推进 running 序列（decode step）
        for seq in self.running:
            if total_tokens + 1 <= self.max_tokens_per_batch:
                scheduled.append(seq)
                total_tokens += 1

        # 2. 空位填充 waiting 序列（prefill step）
        while self.waiting and len(scheduled) < self.max_num_seqs:
            seq = self.waiting.pop(0)
            if total_tokens + seq.prompt_len <= self.max_tokens_per_batch:
                scheduled.append(seq)
                total_tokens += seq.prompt_len
            else:
                self.waiting.insert(0, seq)
                break

        # 3. 移除已完成序列，腾出位置
        self.running = [s for s in scheduled if not s.is_finished()]
        return ScheduledBatch(scheduled)

from vllm import LLM, SamplingParams

# 直接加载 GPTQ 量化模型
llm = LLM(
    model="TheBloke/Llama-2-13B-GPTQ",
    quantization="gptq",
    dtype="float16",
    gpu_memory_utilization=0.9,
    max_model_len=4096,
    enforce_eager=False,  # 使用 CUDA Graph 加速
)

sampling = SamplingParams(temperature=0.7, max_tokens=512)
outputs = llm.generate(["Explain PagedAttention in 3 sentences."], sampling)

# 启动 OpenAI 兼容 API 服务器
python -m vllm.entrypoints.openai.api_server \
  --model TheBloke/Mistral-7B-Instruct-v0.2-AWQ \
  --quantization awq \
  --dtype float16 \
  --gpu-memory-utilization 0.9 \
  --max-model-len 8192 \
  --port 8000 \
  --enable-prefix-caching \
  --enforce-eager

# FP8 精度，H100 上最佳吞吐
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Llama-3.1-70B-Instruct \
  --quantization fp8 \
  --kv-cache-dtype fp8 \
  --gpu-memory-utilization 0.92 \
  --tensor-parallel-size 4

# 4 卡张量并行（推荐用于 70B+ 模型）
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 4 \
  --gpu-memory-utilization 0.9 \
  --max-model-len 32768

# 2 节点 × 4 卡（需要 MPI 或 Ray）
# 节点 0
VLLM_HOST_IP=10.0.0.1 python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Llama-3.1-405B-Instruct \
  --tensor-parallel-size 8 \
  --pipeline-parallel-size 1 \
  --distributed-executor-backend ray

# 张量并行 + 流水线并行混合
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Llama-3.1-405B-Instruct \
  --tensor-parallel-size 4 \
  --pipeline-parallel-size 2 \
  --gpu-memory-utilization 0.9

# 启用前缀缓存（vLLM 0.4+）
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Llama-3.1-8B-Instruct \
  --enable-prefix-caching \
  --gpu-memory-utilization 0.9

version: "3.9"
services:
  vllm:
    image: vllm/vllm-openai:latest
    container_name: vllm
    runtime: nvidia
    environment:
      - NVIDIA_VISIBLE_DEVICES=0,1,2,3
      - HF_TOKEN=${HF_TOKEN}
    volumes:
      - ./models:/root/.cache/huggingface
    ports:
      - "8000:8000"
    command:
      - --model=meta-llama/Llama-3.1-70B-Instruct
      - --tensor-parallel-size=4
      - --gpu-memory-utilization=0.9
      - --max-model-len=32768
      - --enable-prefix-caching
      - --uvicorn-log-level=info
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 5
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]

# 使用 vLLM 内置 Prometheus 指标
# /metrics 端点自动暴露