评估是LLM迭代的指南针

没有评估就没有优化。LLM评估管线是模型迭代的基础设施——它告诉你新版本是变好了还是变差了，哪些能力提升了哪些下降了。

评估维度

EVAL_DIMENSIONS = {
    "knowledge": ["MMLU", "C-Eval", "CMMLU"],        # 知识问答
    "reasoning": ["GSM8K", "MATH", "BBH"],           # 推理能力
    "coding": ["HumanEval", "MBPP", "CodeContests"],  # 代码生成
    "instruction_following": ["IFEval", "MT-Bench"], # 指令跟随
    "safety": ["ToxiGen", "TruthfulQA"],             # 安全性
    "multilingual": ["MGSM", "XNLI"],                # 多语言
}

自动化评估管线

class EvalPipeline:
    def __init__(self, model, benchmarks):
        self.model = model
        self.benchmarks = benchmarks
    
    async def run_all(self):
        results = {}
        for name, benchmark in self.benchmarks.items():
            results[name] = await self.run_benchmark(name, benchmark)
        
        report = self.generate_report(results)
        return report
    
    async def run_benchmark(self, name, benchmark):
        scores = []
        for sample in benchmark.samples:
            response = await self.model.generate(sample["input"])
            score = benchmark.evaluate(response, sample["expected"])
            scores.append(score)
        
        return {
            "benchmark": name,
            "score": sum(scores) / len(scores),
            "n_samples": len(scores),
            "details": scores,
        }

LLM-as-Judge评估

class LLMJudge:
    def __init__(self, judge_model):
        self.judge = judge_model
    
    async def evaluate(self, question, response, reference=None, criteria=None):
        prompt = f"""请评估以下回答的质量。

问题：{question}
回答：{response}
{'参考答案：' + reference if reference else ''}
评估标准：{criteria or '准确性、完整性、清晰度'}

请给出1-10分的评分和理由。
输出JSON格式：{{"score": 8, "reason": "...", "breakdown": {{"accuracy": 8, "completeness": 7, "clarity": 9}}}}"""
        
        result = await self.judge.generate(prompt)
        return json.loads(result)
    
    async def compare(self, question, response_a, response_b):
        """对比两个回答"""
        prompt = f"""比较以下两个回答的优劣。

问题：{question}
回答A：{response_a}
回答B：{response_b}

输出JSON：{{"winner": "A"或"B"或"tie", "reason": "..."}}"""
        
        result = await self.judge.generate(prompt)
        return json.loads(result)

回归测试

class RegressionTester:
    def __init__(self, baseline_results):
        self.baseline = baseline_results
    
    async def check_regression(self, new_results, threshold=0.02):
        """检查是否有性能回归"""
        regressions = []
        
        for benchmark, new_score in new_results.items():
            if benchmark in self.baseline:
                old_score = self.baseline[benchmark]
                delta = new_score["score"] - old_score["score"]
                
                if delta < -threshold:
                    regressions.append({
                        "benchmark": benchmark,
                        "old": old_score["score"],
                        "new": new_score["score"],
                        "delta": delta,
                    })
        
        return regressions

评估报告

def generate_eval_report(results, baseline=None):
    """生成评估报告"""
    report = "# LLM评估报告\n\n"
    report += f"日期：{datetime.now().strftime('%Y-%m-%d')}\n\n"
    
    report += "## 评估结果\n\n"
    report += "| 基准测试 | 得分 | 基线 | 变化 |\n"
    report += "|---------|------|------|------|\n"
    
    for name, result in results.items():
        score = f"{result['score']:.4f}"
        if baseline and name in baseline:
            base = baseline[name]["score"]
            delta = result["score"] - base
            delta_str = f"{'🟢' if delta >= 0 else '🔴'} {delta:+.4f}"
        else:
            base = "-"
            delta_str = "-"
        
        report += f"| {name} | {score} | {base:.4f} | {delta_str} |\n"
    
    if baseline:
        regressions = [r for r in results if baseline.get(r, {}).get("score", 0) - results[r]["score"] > 0.02]
        if regressions:
            report += f"\n## ⚠️ 检测到回归\n\n"
            for r in regressions:
                report += f"- **{r}**: {baseline[r]['score']:.4f} → {results[r]['score']:.4f}\n"
    
    return report

结语

LLM评估管线是模型迭代的质量把关者。自动化基准测试提供客观指标，LLM-as-Judge提供主观评估，回归测试防止质量倒退。建立定期评估机制，确保每次模型更新都有数据支撑。

加入讨论

这篇文章有姊妹讨论帖在硅基AGI论坛 — 全球首个碳基硅基认知交流平台。

碳基与硅基的智慧碰撞，认知差异创造无限可能。