评估数据策划方法：好数据才能评出好模型
引言 评估数据的质量直接决定了评估结果的可信度。“垃圾进，垃圾出"在LLM评估中同样适用。一个有偏见的评估数据集可能让你做出错误的模型选择。2026年，评估数据策划已经成为一门专门的学问。本文将系统介绍评估数据策划的方法论。 评估数据的挑战 挑战一：代表性 评估数据是否代表了真实使用场景？ 问题：评估数据全部来自新闻文本 真实场景：用户输入包含口语、错别字、混合语言 → 评估结果无法反映真实效果 挑战二：多样性 评估数据是否覆盖了各种输入类型？ 问题：情感分析评估集只有明显正面/负面 缺失：讽刺、中性、混合情感 → 模型在边界情况上的表现未知 挑战三：数据污染 评估数据是否泄露到了训练集中？ 挑战四：时效性 评估数据是否过时？ 挑战五：偏见 评估数据是否对某些群体或观点有偏见？ 评估数据策划流程 步骤一：需求分析 eval_data_requirements = { "task": "客服对话质量评估", "dimensions": { "准确性": "回复信息是否正确", "有用性": "是否解决了用户问题", "态度": "回复是否礼貌友好", "效率": "是否在3轮内解决" }, "coverage": { "query_types": ["咨询", "投诉", "退款", "技术支持"], "difficulty": ["简单", "中等", "困难"], "languages": ["中文"], "domains": ["电商", "金融", "教育"] }, "size": { "minimum": 500, "recommended": 2000, "ideal": 5000 } } 步骤二：数据收集 来源一：真实用户数据 def collect_real_user_data(production_logs, n=1000): """ 从生产日志中采样真实用户数据 """ # 随机采样 sampled = random.sample(production_logs, min(n, len(production_logs))) # 去敏处理 sanitized = [sanitize(data) for data in sampled] # 分类标注 categorized = categorize(sanitized) return categorized 来源二：人工构造 def generate_synthetic_data(task_description, n=500): """ 用LLM生成合成评估数据 """ prompt = f""" 请为以下任务生成{n}个评估用例： 任务：{task_description} 要求： 1. 覆盖不同难度（简单/中等/困难） 2. 包含边界情况 3. 包含对抗性输入 4. 输入多样化 以JSON格式输出。 """ return call_llm(prompt) 来源三：专家标注 def expert_annotation(raw_data, experts): """ 邀请领域专家标注数据 """ annotated = [] for item in raw_data: # 3位专家独立标注 labels = [expert.annotate(item) for expert in experts[:3]] # 计算一致性 agreement = compute_agreement(labels) if agreement > 0.8: # 一致性高，取多数意见 item["label"] = majority_vote(labels) else: # 一致性低，讨论后决定 item["label"] = expert_discussion(item, labels) annotated.append(item) return annotated 步骤三：数据清洗 def clean_eval_data(data): """ 清洗评估数据 """ cleaned = [] for item in data: # 去重 if is_duplicate(item, cleaned): continue # 去除敏感信息 item = remove_pii(item) # 检查标注质量 if not validate_annotation(item): continue # 检查输入长度 if len(item["input"]) < 5 or len(item["input"]) > 10000: continue cleaned.append(item) return cleaned 步骤四：数据分析 def analyze_eval_data(data): """ 分析评估数据集的分布 """ analysis = { "size": len(data), "difficulty_distribution": Counter(d["difficulty"] for d in data), "category_distribution": Counter(d["category"] for d in data), "length_distribution": [len(d["input"]) for d in data], "language_distribution": Counter(d["language"] for d in data), "bias_check": check_bias(data), "diversity_score": compute_diversity(data), "contamination_check": check_contamination(data) } return analysis 步骤五：数据平衡 def balance_eval_data(data, target_distribution=None): """ 平衡评估数据集 """ if target_distribution is None: # 默认均衡分布 categories = set(d["category"] for d in data) target_distribution = {c: 1/len(categories) for c in categories} # 按类别分组 by_category = defaultdict(list) for d in data: by_category[d["category"]].append(d) # 计算每个类别的目标数量 total = len(data) balanced = [] for category, items in by_category.items(): target_count = int(total * target_distribution[category]) if len(items) > target_count: # 过采样 sampled = random.sample(items, target_count) else: # 欠采样或数据增强 sampled = items + augment_data(items, target_count - len(items)) balanced.extend(sampled) return balanced 数据质量评估 评估维度 def evaluate_data_quality(data): """ 评估数据集质量 """ quality = {} # 1. 覆盖度：是否覆盖了所有需要评估的场景 quality["coverage"] = evaluate_coverage(data) # 2. 多样性：输入是否足够多样 quality["diversity"] = evaluate_diversity(data) # 3. 标注一致性：标注是否可靠 quality["annotation_consistency"] = evaluate_consistency(data) # 4. 偏见检测：是否存在偏见 quality["bias"] = detect_bias(data) # 5. 污染检测：是否与训练数据重叠 quality["contamination"] = detect_contamination(data) # 6. 难度分布：难度是否合理 quality["difficulty"] = evaluate_difficulty(data) return quality 偏见检测 def detect_bias(data): """ 检测数据集中的偏见 """ biases = [] # 性别偏见 male_terms = ["他", "男性", "先生"] female_terms = ["她", "女性", "女士"] male_count = sum(1 for d in data if any(t in d["input"] for t in male_terms)) female_count = sum(1 for d in data if any(t in d["input"] for t in female_terms)) if abs(male_count - female_count) / max(male_count, female_count) > 0.3: biases.append(f"性别分布不均：男性{male_count}，女性{female_count}") # 地域偏见 # 年龄偏见 # 领域偏见 return biases 污染检测 def detect_contamination(eval_data, train_data_sample): """ 检测评估数据是否泄露到训练集中 """ contamination = [] for eval_item in eval_data: # 精确匹配 for train_item in train_data_sample: if eval_item["input"] == train_item: contamination.append(eval_item) break # 模糊匹配（n-gram重叠） eval_ngrams = set(get_ngrams(eval_item["input"], 8)) for train_item in train_data_sample: train_ngrams = set(get_ngrams(train_item, 8)) overlap = len(eval_ngrams & train_ngrams) / len(eval_ngrams) if overlap > 0.8: contamination.append(eval_item) break return { "contaminated_count": len(contamination), "contamination_rate": len(contamination) / len(eval_data), "contaminated_items": contamination } 评估数据管理 版本管理 class EvalDatasetVersion: def __init__(self, name, version, data): self.name = name self.version = version self.data = data self.created_at = datetime.now() self.hash = compute_hash(data) def diff(self, other_version): """计算版本差异""" added = [d for d in self.data if d not in other_version.data] removed = [d for d in other_version.data if d not in self.data] return {"added": added, "removed": removed} 数据集文档 # eval_dataset_card.yaml name: "客服对话评估集 v2.0" version: "2.0" created: "2026-06-01" size: 2000 description: "电商客服对话质量评估数据集" coverage: query_types: 咨询: 500 投诉: 400 退款: 350 技术支持: 350 其他: 400 difficulty: 简单: 600 中等: 900 困难: 500 quality: annotation_consistency: 0.87 diversity_score: 0.82 bias_check: "通过" contamination_check: "无污染" limitation: - "仅覆盖电商领域" - "中文数据为主" - "不含多轮对话" 2026年新趋势 1. 动态评估数据 评估数据集定期更新，防止数据污染和过时。 ...
