vLLM 生产部署指南：高吞吐推理引擎

┌─────────────────────────────────────────┐
│  连续内存分配（传统方式）                  │
│  ┌──────┐┌──────┐┌──────┐┌──────────┐ │
│  │Req A ││Req B ││Req C ││  空洞    │ │
│  │2GB   ││2GB   ││2GB   ││  1.5GB  │ │
│  └──────┘└──────┘└──────┘└──────────┘ │
│  问题：外部碎片导致 1.5GB 无法利用       │
└─────────────────────────────────────────┘

┌─────────────────────────────────────────┐
│  分页内存分配（PagedAttention）          │
│  ┌────┐┌────┐┌────┐┌────┐┌────┐┌────┐│
│  │A-1 ││A-2 ││B-1 ││C-1 ││B-2 ││C-2 ││
│  │16KB││16KB││16KB││16KB││16KB││16KB││
│  └────┘└────┘└────┘└────┘└────┘└────┘│
│  KV Cache 以块为单位，物理非连续逻辑连续 │
│  利用率：96%+                            │
└─────────────────────────────────────────┘

# vLLM 连续批处理示意
# 时间轴 →
# 传统批处理：
# [Req1, Req2, Req3] ──────────── [全部完成] → [Req4, Req5, ...]
#                          ↑ GPU 空闲等待慢请求

# 连续批处理：
# [Req1, Req2, Req3] ──── [Req1完成, 加入Req4] ──── [Req2完成, 加入Req5]
#  ↑ 每个迭代步都可以动态调整批次成员，GPU 始终满载

# 基础安装
pip install vllm

# 带 OpenAI 兼容 API
pip install "vllm[openai]"

# 从源码安装（最新功能）
pip install vllm --pre --extra-index-url https://wheels.vllm.ai/nightly

# 启动 API 服务器
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Llama-3.3-70B-Instruct \
  --tensor-parallel-size 4 \
  --gpu-memory-utilization 0.9 \
  --max-model-len 8192 \
  --port 8000

# Dockerfile
FROM vllm/vllm-openai:latest

# 复制自定义配置
COPY config.yaml /app/config.yaml

# 设置环境变量
ENV VLLM_API_KEY=your-secret-key
ENV HF_TOKEN=your-hf-token

EXPOSE 8000

CMD ["--model", "meta-llama/Llama-3.3-70B-Instruct", \
     "--tensor-parallel-size", "4", \
     "--gpu-memory-utilization", "0.9"]

# docker-compose.yml
version: '3.8'

services:
  vllm:
    image: vllm/vllm-openai:latest
    container_name: vllm-server
    runtime: nvidia
    environment:
      - NVIDIA_VISIBLE_DEVICES=0,1,2,3
      - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN}
    volumes:
      - model_cache:/root/.cache/huggingface
      - ./config.yaml:/app/config.yaml
    ports:
      - "8000:8000"
    command:
      - --model=meta-llama/Llama-3.3-70B-Instruct
      - --tensor-parallel-size=4
      - --gpu-memory-utilization=0.9
      - --max-model-len=8192
      - --enable-prefix-caching
      - --api-key=${VLLM_API_KEY}
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 4
              capabilities: [gpu]

volumes:
  model_cache:

apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-server
  labels:
    app: vllm
spec:
  replicas: 2
  selector:
    matchLabels:
      app: vllm
  template:
    metadata:
      labels:
        app: vllm
    spec:
      containers:
      - name: vllm
        image: vllm/vllm-openai:latest
        args:
          - --model=meta-llama/Llama-3.3-70B-Instruct
          - --tensor-parallel-size=4
          - --gpu-memory-utilization=0.9
          - --max-model-len=8192
          - --enable-prefix-caching
        resources:
          limits:
            nvidia.com/gpu: 4
            memory: 128Gi
          requests:
            nvidia.com/gpu: 4
            memory: 64Gi
        ports:
        - containerPort: 8000
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 120
          periodSeconds: 10
        livenessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 300
          periodSeconds: 30
---
apiVersion: v1
kind: Service
metadata:
  name: vllm-service
spec:
  selector:
    app: vllm
  ports:
  - port: 80
    targetPort: 8000
  type: LoadBalancer

from openai import OpenAI

# 连接 vLLM 服务器
client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="your-secret-key",
)

# 聊天补全
response = client.chat.completions.create(
    model="meta-llama/Llama-3.3-70B-Instruct",
    messages=[
        {"role": "system", "content": "你是技术专家。"},
        {"role": "user", "content": "解释 PagedAttention 原理"},
    ],
    max_tokens=1024,
    temperature=0.7,
    stream=True,  # 流式输出
)

for chunk in response:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="")

from vllm import LLM, SamplingParams

# 初始化引擎
llm = LLM(
    model="meta-llama/Llama-3.3-70B-Instruct",
    tensor_parallel_size=4,
    gpu_memory_utilization=0.9,
    max_model_len=8192,
    enable_prefix_caching=True,
)

# 采样参数
sampling_params = SamplingParams(
    temperature=0.7,
    top_p=0.9,
    max_tokens=1024,
)

# 批量推理
prompts = [
    "用 Python 实现二叉搜索树",
    "解释什么是分布式系统中的 CAP 定理",
    "写一首关于编程的诗",
    "对比 React 和 Vue 的优缺点",
]

outputs = llm.generate(prompts, sampling_params)

for output in outputs:
    prompt = output.prompt
    generated_text = output.outputs[0].text
    print(f"Prompt: {prompt[:50]}...")
    print(f"Output: {generated_text[:100]}...")
    print(f"Tokens: {len(output.outputs[0].token_ids)}")
    print("---")

python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Llama-3.3-70B-Instruct \
  --enable-prefix-caching \
  --prefix-cache-hit-cpu-time-ratio=0.1

# prometheus.yml
scrape_configs:
  - job_name: 'vllm'
    static_configs:
      - targets: ['vllm-server:8000']
    metrics_path: '/metrics'

{
  "dashboard": {
    "title": "vLLM 推理服务监控",
    "panels": [
      {
        "title": "请求吞吐量",
        "query": "rate(vllm:num_requests_running[1m])"
      },
      {
        "title": "GPU 缓存使用率",
        "query": "vllm:gpu_cache_usage_perc",
        "thresholds": [{"value": 0.9, "color": "red"}]
      }
    ]
  }
}

# 降低 GPU 内存利用率
--gpu-memory-utilization 0.85

# 减小最大上下文长度
--max-model-len 4096

# 减少并发序列数
--max-num-seqs 128

# 确保 HuggingFace token 配置正确
export HF_TOKEN=your_token

# 使用镜像下载
export HF_ENDPOINT=https://hf-mirror.com

# 指定本地模型路径
--model /data/models/Llama-3.3-70B-Instruct