大模型评估方法论：从基准测试到人类偏好的全面评估体系

EVALUATION_DIMENSIONS = {
    "知识能力": {
        "MMLU-Pro": "多任务语言理解（学术知识）",
        "C-Eval": "中文综合能力",
        "BBH": "BIG-Bench Hard（推理）",
        "TruthfulQA": "真实性评估"
    },
    "推理能力": {
        "GSM8K": "小学数学推理",
        "MATH": "高等数学推理",
        "GPQA": "研究生水平问答",
        "ARC": "科学推理"
    },
    "代码能力": {
        "HumanEval": "Python代码生成",
        "MBPP": "基础编程",
        "SWE-bench": "软件工程任务",
        "LiveCodeBench": "实时编程竞赛"
    },
    "语言能力": {
        "MT-Bench": "多轮对话",
        "AlpacaEval": "指令跟随",
        "IFEval": "指令执行评估"
    },
    "安全对齐": {
        "AdvBench": "对抗性提示",
        "HarmBench": "有害行为测试",
        "BBQ": "偏见评估"
    }
}

class BenchmarkRunner:
    def __init__(self, model, config):
        self.model = model
        self.config = config
    
    def run_all(self):
        results = {}
        for bench_name, bench_class in BENCHMARKS.items():
            results[bench_name] = self._run_benchmark(bench_name, bench_class)
        return results
    
    def _run_benchmark(self, name, bench_class):
        benchmark = bench_class()
        
        # 多次运行取平均（降低随机性）
        scores = []
        for run in range(self.config.get("n_runs", 1)):
            score = self._single_run(benchmark)
            scores.append(score)
        
        return {
            "benchmark": name,
            "scores": scores,
            "mean": np.mean(scores),
            "std": np.std(scores),
            "details": self._collect_details(benchmark)
        }
    
    def _single_run(self, benchmark):
        correct = 0
        for question in benchmark.questions:
            response = self.model.generate(
                question.prompt,
                temperature=0.0,  # 贪婪解码，确保可复现
                max_tokens=question.max_tokens
            )
            
            if benchmark.check_answer(response, question.answer):
                correct += 1
        
        return correct / len(benchmark.questions)

class EvaluationPitfalls:
    pitfalls = {
        "数据污染": {
            "description": "测试集出现在训练数据中",
            "detection": "检查测试问题是否在训练数据中出现",
            "mitigation": "使用动态更新的测试集，如LiveCodeBench"
        },
        "格式敏感性": {
            "description": "模型答案正确但格式不匹配",
            "detection": "人工检查错误样本",
            "mitigation": "使用灵活的答案匹配（正则/语义匹配）"
        },
        "位置偏差": {
            "description": "多选题中模型偏好某些位置",
            "detection": "打乱选项顺序重新测试",
            "mitigation": "多次测试取平均"
        },
        "提示敏感性": {
            "description": "不同prompt模板导致分数差异大",
            "detection": "用多种prompt模板测试",
            "mitigation": "报告多个模板的平均分"
        }
    }

class LLMJudge:
    def __init__(self, judge_model="gpt-4o"):
        self.judge = judge_model
    
    def evaluate(self, question, response_a, response_b):
        """用强模型评估两个回答的优劣"""
        prompt = f"""
        请评估以下两个回答的质量。
        
        问题：{question}
        
        回答A：{response_a}
        回答B：{response_b}
        
        评估维度（1-10分）：
        1. 准确性：信息是否正确
        2. 完整性：是否充分回答了问题
        3. 清晰度：表达是否清晰易懂
        4. 有用性：对提问者是否有帮助
        
        输出JSON：
        {{
            "A": {{"accuracy": X, "completeness": X, "clarity": X, "helpfulness": X}},
            "B": {{"accuracy": X, "completeness": X, "clarity": X, "helpfulness": X}},
            "winner": "A" | "B" | "tie",
            "reasoning": "..."
        }}
        """
        
        return self.judge.generate(prompt)
    
    def evaluate_with_rubric(self, question, response, rubric):
        """基于评分标准的评估"""
        prompt = f"""
        按以下评分标准评估回答：
        
        问题：{question}
        回答：{response}
        
        评分标准：
        {rubric}
        
        对每个标准给出1-5分和具体理由。
        """
        
        return self.judge.generate(prompt)

class HumanEvaluation:
    def __init__(self):
        self.evaluators = []
        self.tasks = []
    
    def setup_eval(self, questions, responses, criteria):
        """设置人类评估任务"""
        for q, responses_pair in zip(questions, responses):
            self.tasks.append({
                "question": q,
                "response_a": responses_pair[0],
                "response_b": responses_pair[1],
                "criteria": criteria
            })
    
    def collect_ratings(self):
        """收集人类评估结果"""
        results = []
        for task in self.tasks:
            # 呈现给评估者
            rating = self._present_to_evaluator(task)
            results.append(rating)
        
        # 计算一致性
        agreement = self._compute_inter_annotator_agreement(results)
        
        return {
            "results": results,
            "inter_annotator_agreement": agreement,
            "elo_ratings": self._compute_elo(results)
        }
    
    def _compute_inter_annotator_agreement(self, results):
        """计算评估者间一致性"""
        from sklearn.metrics import cohen_kappa_score
        # 如果一致性<0.6，说明评估标准需要改进
        return cohen_kappa_score(results[0], results[1])

class EloRatingSystem:
    def __init__(self, k=32):
        self.k = k
        self.ratings = {}  # model_name -> elo rating
    
    def update(self, model_a, model_b, result):
        """根据对战结果更新Elo分"""
        ra = self.ratings.get(model_a, 1200)
        rb = self.ratings.get(model_b, 1200)
        
        # 预期胜率
        ea = 1 / (1 + 10 ** ((rb - ra) / 400))
        eb = 1 - ea
        
        # 实际结果
        if result == "A":
            sa, sb = 1, 0
        elif result == "B":
            sa, sb = 0, 1
        else:  # tie
            sa, sb = 0.5, 0.5
        
        # 更新分数
        self.ratings[model_a] = ra + self.k * (sa - ea)
        self.ratings[model_b] = rb + self.k * (sb - eb)
    
    def get_rankings(self):
        return sorted(self.ratings.items(), key=lambda x: x[1], reverse=True)

class CodeEvaluation:
    def evaluate(self, model, problems):
        """代码生成评估"""
        results = {
            "pass@1": 0,
            "pass@10": 0,
            "pass@100": 0,
            "syntax_error_rate": 0,
            "runtime_error_rate": 0
        }
        
        for problem in problems:
            # 生成多个解决方案
            solutions = [model.generate(problem.prompt) for _ in range(100)]
            
            # 逐个测试
            passed = 0
            for solution in solutions:
                result = self._run_tests(solution, problem.test_cases)
                if result["passed"]:
                    passed += 1
                elif result["error_type"] == "syntax":
                    results["syntax_error_rate"] += 1
                elif result["error_type"] == "runtime":
                    results["runtime_error_rate"] += 1
            
            # pass@k
            results["pass@1"] += passed > 0
            results["pass@10"] += passed > 10
            results["pass@100"] += passed > 0
        
        # 归一化
        n = len(problems)
        for k in ["pass@1", "pass@10", "pass@100"]:
            results[k] /= n
        for k in ["syntax_error_rate", "runtime_error_rate"]:
            results[k] /= (n * 100)
        
        return results

class SafetyEvaluation:
    def __init__(self):
        self.attack_prompts = self._load_attack_prompts()
    
    def evaluate(self, model):
        """安全评估"""
        results = {
            "jailbreak_success": 0,
            "harmful_content_generated": 0,
            "bias_detected": 0,
            "pii_leaked": 0
        }
        
        # 越狱测试
        for attack in self.attack_prompts["jailbreak"]:
            response = model.generate(attack["prompt"])
            if self._is_jailbreak_successful(response, attack["target"]):
                results["jailbreak_success"] += 1
        
        # 有害内容测试
        for prompt in self.attack_prompts["harmful"]:
            response = model.generate(prompt)
            if self._is_harmful(response):
                results["harmful_content_generated"] += 1
        
        # 偏见测试
        for prompt in self.attack_prompts["bias"]:
            response = model.generate(prompt)
            bias_score = self._measure_bias(response)
            if bias_score > 0.5:
                results["bias_detected"] += 1
        
        total = len(self.attack_prompts["jailbreak"])
        for k in results:
            results[k] = {"count": results[k], "rate": results[k] / total}
        
        return results

class EvaluationReportGenerator:
    def generate(self, model_name, results):
        """生成综合评估报告"""
        return f"""
# {model_name} 评估报告

## 综合评分
- 知识能力: {results['knowledge']['mean']:.1f}/100
- 推理能力: {results['reasoning']['mean']:.1f}/100
- 代码能力: {results['coding']['pass@1']*100:.1f}%
- 对话能力: {results['dialogue']['elo']:.0f} Elo
- 安全性: {results['safety']['safe_rate']*100:.1f}%

## 详细分析

### 优势
{self._format_strengths(results)}

### 弱项
{self._format_weaknesses(results)}

### 与其他模型对比
{self._format_comparison(model_name, results)}

### 数据污染检查
{self._contamination_report(results)}

## 结论
{self._conclusion(results)}
"""