对齐技术全景 预训练模型 (Base) ↓ SFT (Supervised Fine-Tuning) ← 监督微调，学习指令遵循 ↓ DPO / RLHF ← 偏好对齐，学习人类价值观 ↓ 对齐模型 (Chat/Instruct) 阶段 目标 数据 方法 SFT 学会跟随指令 (指令, 回答) 对 监督学习 RLHF 对齐人类偏好 (prompt, chosen, rejected) 强化学习 DPO 对齐人类偏好 同上 直接优化，无需 RL SFT：监督微调 数据格式 Alpaca 格式：
{ "instruction": "将以下句子翻译为英文", "input": "今天天气真好", "output": "The weather is really nice today." } ShareGPT 格式（多轮对话）：
{ "conversations": [ {"from": "human", "value": "你好"}, {"from": "gpt", "value": "你好！有什么可以帮你的？"}, {"from": "human", "value": "解释一下量子计算"}, {"from": "gpt", "value": "量子计算利用量子力学原理..."} ] } 数据构造 import json import random class SFTDataBuilder: def __init__(self, llm_call): self.llm = llm_call def generate_alpaca_style(self, seed_topics: list[str], n_samples: int = 1000): """用 LLM 自动生成 SFT 数据""" dataset = [] for topic in seed_topics: prompt = f"""生成 10 条多样化的指令-回答对。 主题：{topic} 要求： 1. 指令类型多样（问答、翻译、摘要、代码、推理等） 2. 难度从简单到复杂 3. 回答准确、自然 输出 JSON 数组格式： [{{"instruction": "", "input": "", "output": ""}}]""" result = self.llm(prompt) try: items = json.loads(result) dataset.extend(items) except json.JSONDecodeError: continue # 去重 + 打乱 seen = set() unique = [] for item in dataset: key = item["instruction"] + item.get("input", "") if key not in seen: seen.add(key) unique.append(item) random.shuffle(unique) return unique[:n_samples] def to_sharegpt(self, alpaca_data: list[dict]) -> list[dict]: """Alpaca → ShareGPT 格式转换""" converted = [] for item in alpaca_data: instruction = item["instruction"] if item.get("input"): instruction = f"{instruction}\n\n输入: {item['input']}" converted.append({ "conversations": [ {"from": "human", "value": instruction}, {"from": "gpt", "value": item["output"]}, ] }) return converted 训练配置 from transformers import ( AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer, ) from datasets import Dataset from peft import LoraConfig, get_peft_model, TaskType import torch class SFTTrainer: def __init__(self, model_path: str, use_lora: bool = True): self.tokenizer = AutoTokenizer.from_pretrained(model_path) self.tokenizer.pad_token = self.tokenizer.eos_token self.model = AutoModelForCausalLM.from_pretrained( model_path, torch_dtype=torch.bfloat16, device_map="auto", ) if use_lora: self._setup_lora() def _setup_lora(self): """配置 LoRA：只训练少量参数""" lora_config = LoraConfig( task_type=TaskType.CAUSAL_LM, r=16, # LoRA rank lora_alpha=32, # 缩放因子 lora_dropout=0.05, target_modules=[ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", ], ) self.model = get_peft_model(self.model, lora_config) self.model.print_trainable_parameters() # 输出: trainable params: 13M || all params: 7B || trainable%: 0.19% def format_prompt(self, instruction: str, input_text: str = "", output: str = "") -> str: """格式化为 ChatML 模板""" user_msg = instruction if input_text: user_msg += f"\n\n{input_text}" prompt = f"<|im_start|>user\n{user_msg}<|im_end|>\n" if output: prompt += f"<|im_start|>assistant\n{output}<|im_end|>" return prompt def train(self, data: list[dict], learning_rate: float = 2e-5, num_epochs: int = 3, batch_size: int = 4, max_length: int = 1024): # 格式化 texts = [ self.format_prompt( d["instruction"], d.get("input", ""), d["output"] ) for d in data ] dataset = Dataset.from_dict({"text": texts}) def tokenize(examples): tokenized = self.tokenizer( examples["text"], truncation=True, max_length=max_length, padding="max_length", ) tokenized["labels"] = tokenized["input_ids"].copy() return tokenized dataset = dataset.map(tokenize, batched=True) training_args = TrainingArguments( output_dir="./sft_output", num_train_epochs=num_epochs, per_device_train_batch_size=batch_size, gradient_accumulation_steps=4, learning_rate=learning_rate, warmup_ratio=0.03, lr_scheduler_type="cosine", logging_steps=10, save_strategy="epoch", bf16=True, optim="adamw_torch", ) trainer = Trainer( model=self.model, args=training_args, train_dataset=dataset, ) trainer.train() LoRA 超参数选择 参数 推荐值 说明 r (rank) 8-64 越大能力越强，但过拟合风险 lora_alpha 2×r 经验值，平衡缩放 target_modules all linear 全覆盖效果最好 dropout 0.05 防过拟合 learning_rate 1e-4 ~ 5e-5 LoRA 可用较大学习率 DPO：直接偏好优化 DPO (Rafailov et al., 2023) 是 RLHF 的简化替代——无需训练奖励模型，无需强化学习：
...
