模型量化部署指南

是否在H100上部署？
├─ 是 → FP8量化（原生加速，精度最佳）
└─ 否 → 显存是否足够FP16？
         ├─ 是 → FP16（无精度损失）
         └─ 否 → INT8还是INT4？
                  ├─ 精度要求高 → INT8 (W8A8)
                  └─ 显存优先 → INT4 (GPTQ/AWQ)

# 使用AutoGPTQ进行量化
from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig

# 量化配置
quantize_config = BaseQuantizeConfig(
    bits=4,              # 4-bit量化
    group_size=128,      # 分组大小
    desc_act=False,      # 是否按激活降序排列
)

# 准备校准数据
calibration_data = load_calibration_dataset(n_samples=128, max_length=512)

# 加载模型并量化
model = AutoGPTQForCausalLM.from_pretrained(
    "Qwen/Qwen3-32B",
    quantize_config,
)
model.quantize(calibration_data)

# 保存量化模型
model.save_quantized("./qwen3-32b-gptq-4bit")

# 使用AutoAWQ进行量化
from awq import AutoAWQForCausalLM

model = AutoAWQForCausalLM.from_pretrained("Qwen/Qwen3-32B")

quant_config = {
    "zero_point": True,
    "q_group_size": 128,
    "w_bit": 4,
    "version": "GEMM"
}

model.quantize(
    calibration_data,
    quant_config=quant_config
)
model.save_quantized("./qwen3-32b-awq-4bit")

# GPTQ模型部署
vllm serve ./qwen3-32b-gptq-4bit \
  --quantization gptq \
  --dtype float16 \
  --tensor-parallel-size 1 \
  --max-model-len 8192 \
  --gpu-memory-utilization 0.9

# AWQ模型部署
vllm serve ./qwen3-32b-awq-4bit \
  --quantization awq \
  --dtype float16 \
  --max-model-len 8192

# FP8模型部署（H100）
vllm serve ./model \
  --quantization fp8 \
  --kv-cache-dtype fp8

# 从GGUF文件创建Ollama模型
cat > Modelfile <<EOF
FROM ./qwen3-32b-q4_k_m.gguf
PARAMETER num_ctx 8192
PARAMETER temperature 0.7
EOF

ollama create my-qwen -f Modelfile
ollama run my-qwen

def prepare_calibration_data(domain_texts, n_samples=128, max_length=512):
    """准备量化校准数据"""
    samples = []
    
    for text in domain_texts:
        # 截取合适长度的片段
        tokens = tokenizer.encode(text)
        
        for i in range(0, len(tokens) - max_length, max_length // 2):
            chunk = tokens[i:i + max_length]
            samples.append(tokenizer.decode(chunk))
            
            if len(samples) >= n_samples:
                break
        
        if len(samples) >= n_samples:
            break
    
    return samples

async def eval_quantized_model(model_path, benchmarks):
    """评估量化模型的性能"""
    results = {}
    
    for bench_name, bench in benchmarks.items():
        original_score = bench.load_baseline("fp16")
        quantized_score = await bench.evaluate(model_path)
        
        results[bench_name] = {
            "original": original_score,
            "quantized": quantized_score,
            "degradation": (original_score - quantized_score) / original_score * 100
        }
        
        if results[bench_name]["degradation"] > 5:
            print(f"⚠️ {bench_name}: {results[bench_name]['degradation']:.1f}% degradation")
    
    return results