Hermes 4微调实战：从数据准备到模型部署全流程

{
  "messages": [
    {
      "role": "system",
      "content": "你是企业客服助手。"
    },
    {
      "role": "user",
      "content": "订单2024001什么时候发货？"
    },
    {
      "role": "assistant",
      "content": "",
      "tool_calls": [
        {
          "id": "call_001",
          "type": "function",
          "function": {
            "name": "query_order",
            "arguments": "{\"order_id\": \"2024001\"}"
          }
        }
      ]
    },
    {
      "role": "tool",
      "tool_call_id": "call_001",
      "content": "{\"status\": \"已付款\", \"ship_date\": \"2026-07-09\"}"
    },
    {
      "role": "assistant",
      "content": "您的订单2024001已付款，预计7月9日发货。"
    }
  ]
}

class TrainingDataBuilder:
    def __init__(self):
        self.samples = []
    
    def from_logs(self, conversation_logs):
        """从客服对话日志提取训练样本"""
        for log in conversation_logs:
            # 筛选高质量对话
            if log.resolution == "success" and log.satisfaction >= 4:
                sample = self.format_conversation(log)
                self.samples.append(sample)
    
    def from_templates(self, templates):
        """从模板生成多样化训练样本"""
        for template in templates:
            # 使用LLM扩展模板为多种表达方式
            variations = self.expand_template(template, n=10)
            self.samples.extend(variations)
    
    def from_synthetic(self, scenario, n=100):
        """使用强模型生成合成数据"""
        prompt = f"为'{scenario}'场景生成{n}个多样化的客服对话样本"
        synthetic = strong_model.generate(prompt)
        self.samples.extend(self.validate(synthetic))
    
    def build(self):
        """构建训练集"""
        # 去重
        self.samples = self.deduplicate(self.samples)
        # 质量过滤
        self.samples = self.filter_quality(self.samples)
        # 划分训练/验证集
        return self.split(self.samples, ratio=0.95)

def quality_check(sample):
    checks = [
        len(sample["messages"]) >= 3,          # 至少3轮
        has_system_prompt(sample),              # 有系统提示
        tool_calls_valid(sample),               # 工具调用格式正确
        response_length_reasonable(sample),     # 响应长度合理
        no_sensitive_info(sample),              # 无敏感信息
        function_args_match_schema(sample),     # 参数匹配schema
    ]
    return all(checks)

# 硬件：A100 80GB 或 2x RTX 4090
# 软件：
pip install torch transformers peft trl accelerate bitsandbytes

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
from trl import SFTTrainer, SFTConfig

# 加载模型
model_id = "NousResearch/Hermes-4-14B"
tokenizer = AutoTokenizer.from_pretrained(model_id)

# 4-bit量化加载（节省显存）
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    quantization_config=BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_compute_dtype=torch.bfloat16,
        bnb_4bit_quant_type="nf4",
    ),
    device_map="auto"
)
model = prepare_model_for_kbit_training(model)

# LoRA配置
lora_config = LoraConfig(
    r=64,                    # LoRA秩
    lora_alpha=128,          # 缩放因子
    target_modules=[
        "q_proj", "k_proj", "v_proj", "o_proj",
        "gate_proj", "up_proj", "down_proj"
    ],
    lora_dropout=0.05,
    task_type="CAUSAL_LM",
    modules_to_save=["embed_tokens", "lm_head"]  # 保存嵌入层
)
model = get_peft_model(model, lora_config)

# 训练数据
train_dataset = load_dataset("json", data_files="train.jsonl")
eval_dataset = load_dataset("json", data_files="eval.jsonl")

# 训练配置
training_args = SFTConfig(
    output_dir="./hermes-4-finetuned",
    num_train_epochs=3,
    per_device_train_batch_size=4,
    per_device_eval_batch_size=4,
    gradient_accumulation_steps=4,
    warmup_ratio=0.03,
    learning_rate=2e-4,
    lr_scheduler_type="cosine",
    logging_steps=10,
    eval_strategy="steps",
    eval_steps=100,
    save_steps=200,
    bf16=True,
    gradient_checkpointing=True,
    max_seq_length=4096,
)

# 训练器
trainer = SFTTrainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset["train"],
    eval_dataset=eval_dataset["train"],
    tokenizer=tokenizer,
)

# 开始训练
trainer.train()

# 保存
trainer.save_model("./hermes-4-finetuned")

# 关键指标
metrics_to_watch = {
    "train_loss": "应持续下降",
    "eval_loss": "应跟随train_loss下降",
    "eval_loss > train_loss + 0.5": "过拟合警告",
    "learning_rate": "按cosine衰减",
    "grad_norm": "应在1-10范围内",
}

# 典型训练曲线（3 epochs）
# Epoch 0.5: train_loss=1.8, eval_loss=1.9
# Epoch 1.0: train_loss=1.2, eval_loss=1.3
# Epoch 1.5: train_loss=0.9, eval_loss=1.1
# Epoch 2.0: train_loss=0.7, eval_loss=0.9
# Epoch 2.5: train_loss=0.5, eval_loss=0.85
# Epoch 3.0: train_loss=0.4, eval_loss=0.82 ← 最佳

class ModelEvaluator:
    def evaluate(self, model, test_set):
        results = {}
        
        # 函数调用准确率
        results["tool_call_acc"] = self.eval_tool_calls(model, test_set)
        
        # 参数匹配率
        results["param_match"] = self.eval_params(model, test_set)
        
        # 多轮对话一致性
        results["multi_turn"] = self.eval_multi_turn(model, test_set)
        
        # 错误恢复能力
        results["error_recovery"] = self.eval_error_handling(model, test_set)
        
        # 语气/风格一致性
        results["style"] = self.eval_style(model, test_set)
        
        return results

症状：eval_loss开始上升
解决：
- 减少epoch（3→2）
- 增加dropout（0.05→0.1）
- 增加训练数据
- 使用early stopping

症状：微调后函数调用反而变差
解决：
- 确保训练数据中函数调用样本占比>40%
- 检查函数调用格式一致性
- 增加负样本（不需要函数调用的场景）

症状：微调后通用能力下降
解决：
- 混合通用数据（10-20%通用对话）
- 降低learning_rate（2e-4→1e-4）
- 减少LoRA rank（64→32）

from peft import PeftModel

# 加载基础模型
base_model = AutoModelForCausalLM.from_pretrained(
    "NousResearch/Hermes-4-14B",
    torch_dtype=torch.float16,
    device_map="auto"
)

# 加载LoRA
model = PeftModel.from_pretrained(base_model, "./hermes-4-finetuned")

# 合并
model = model.merge_and_unload()

# 保存完整模型
model.save_pretrained("./hermes-4-merged")

# 使用AutoAWQ量化
python -m autoawq \
  --model ./hermes-4-merged \
  --quant awq \
  --bits 4 \
  --output ./hermes-4-awq

# 部署为OpenAI兼容API
python -m vllm.entrypoints.openai.api_server \
  --model ./hermes-4-awq \
  --quantization awq \
  --max-model-len 32768 \
  --gpu-memory-utilization 0.9 \
  --port 8000

生产部署 → 收集用户反馈 → 标注高质量样本 
    ↑                                    ↓
重新训练 ← 评估验证 ← 数据清洗 ← 筛选优质数据

# 模型版本管理
hermes-4-v1.0/     # 初始微调
hermes-4-v1.1/     # 增加边缘场景
hermes-4-v1.2/     # 修复函数调用退化
hermes-4-v2.0/     # 新增10个API函数

# A/B测试
v1.2 → 80%流量
v2.0 → 20%流量
对比指标：解决率、用户满意度、函数调用准确率