评估数据策划方法：好数据才能评出好模型

问题：评估数据全部来自新闻文本
真实场景：用户输入包含口语、错别字、混合语言
→ 评估结果无法反映真实效果

问题：情感分析评估集只有明显正面/负面
缺失：讽刺、中性、混合情感
→ 模型在边界情况上的表现未知

eval_data_requirements = {
    "task": "客服对话质量评估",
    "dimensions": {
        "准确性": "回复信息是否正确",
        "有用性": "是否解决了用户问题",
        "态度": "回复是否礼貌友好",
        "效率": "是否在3轮内解决"
    },
    "coverage": {
        "query_types": ["咨询", "投诉", "退款", "技术支持"],
        "difficulty": ["简单", "中等", "困难"],
        "languages": ["中文"],
        "domains": ["电商", "金融", "教育"]
    },
    "size": {
        "minimum": 500,
        "recommended": 2000,
        "ideal": 5000
    }
}

def collect_real_user_data(production_logs, n=1000):
    """
    从生产日志中采样真实用户数据
    """
    # 随机采样
    sampled = random.sample(production_logs, min(n, len(production_logs)))
    
    # 去敏处理
    sanitized = [sanitize(data) for data in sampled]
    
    # 分类标注
    categorized = categorize(sanitized)
    
    return categorized

def generate_synthetic_data(task_description, n=500):
    """
    用LLM生成合成评估数据
    """
    prompt = f"""
    请为以下任务生成{n}个评估用例：
    任务：{task_description}
    
    要求：
    1. 覆盖不同难度（简单/中等/困难）
    2. 包含边界情况
    3. 包含对抗性输入
    4. 输入多样化
    
    以JSON格式输出。
    """
    
    return call_llm(prompt)

def expert_annotation(raw_data, experts):
    """
    邀请领域专家标注数据
    """
    annotated = []
    for item in raw_data:
        # 3位专家独立标注
        labels = [expert.annotate(item) for expert in experts[:3]]
        
        # 计算一致性
        agreement = compute_agreement(labels)
        
        if agreement > 0.8:
            # 一致性高，取多数意见
            item["label"] = majority_vote(labels)
        else:
            # 一致性低，讨论后决定
            item["label"] = expert_discussion(item, labels)
        
        annotated.append(item)
    
    return annotated

def clean_eval_data(data):
    """
    清洗评估数据
    """
    cleaned = []
    
    for item in data:
        # 去重
        if is_duplicate(item, cleaned):
            continue
        
        # 去除敏感信息
        item = remove_pii(item)
        
        # 检查标注质量
        if not validate_annotation(item):
            continue
        
        # 检查输入长度
        if len(item["input"]) < 5 or len(item["input"]) > 10000:
            continue
        
        cleaned.append(item)
    
    return cleaned

def analyze_eval_data(data):
    """
    分析评估数据集的分布
    """
    analysis = {
        "size": len(data),
        "difficulty_distribution": Counter(d["difficulty"] for d in data),
        "category_distribution": Counter(d["category"] for d in data),
        "length_distribution": [len(d["input"]) for d in data],
        "language_distribution": Counter(d["language"] for d in data),
        "bias_check": check_bias(data),
        "diversity_score": compute_diversity(data),
        "contamination_check": check_contamination(data)
    }
    
    return analysis

def balance_eval_data(data, target_distribution=None):
    """
    平衡评估数据集
    """
    if target_distribution is None:
        # 默认均衡分布
        categories = set(d["category"] for d in data)
        target_distribution = {c: 1/len(categories) for c in categories}
    
    # 按类别分组
    by_category = defaultdict(list)
    for d in data:
        by_category[d["category"]].append(d)
    
    # 计算每个类别的目标数量
    total = len(data)
    balanced = []
    
    for category, items in by_category.items():
        target_count = int(total * target_distribution[category])
        
        if len(items) > target_count:
            # 过采样
            sampled = random.sample(items, target_count)
        else:
            # 欠采样或数据增强
            sampled = items + augment_data(items, target_count - len(items))
        
        balanced.extend(sampled)
    
    return balanced

def evaluate_data_quality(data):
    """
    评估数据集质量
    """
    quality = {}
    
    # 1. 覆盖度：是否覆盖了所有需要评估的场景
    quality["coverage"] = evaluate_coverage(data)
    
    # 2. 多样性：输入是否足够多样
    quality["diversity"] = evaluate_diversity(data)
    
    # 3. 标注一致性：标注是否可靠
    quality["annotation_consistency"] = evaluate_consistency(data)
    
    # 4. 偏见检测：是否存在偏见
    quality["bias"] = detect_bias(data)
    
    # 5. 污染检测：是否与训练数据重叠
    quality["contamination"] = detect_contamination(data)
    
    # 6. 难度分布：难度是否合理
    quality["difficulty"] = evaluate_difficulty(data)
    
    return quality

def detect_bias(data):
    """
    检测数据集中的偏见
    """
    biases = []
    
    # 性别偏见
    male_terms = ["他", "男性", "先生"]
    female_terms = ["她", "女性", "女士"]
    male_count = sum(1 for d in data if any(t in d["input"] for t in male_terms))
    female_count = sum(1 for d in data if any(t in d["input"] for t in female_terms))
    
    if abs(male_count - female_count) / max(male_count, female_count) > 0.3:
        biases.append(f"性别分布不均：男性{male_count}，女性{female_count}")
    
    # 地域偏见
    # 年龄偏见
    # 领域偏见
    
    return biases

def detect_contamination(eval_data, train_data_sample):
    """
    检测评估数据是否泄露到训练集中
    """
    contamination = []
    
    for eval_item in eval_data:
        # 精确匹配
        for train_item in train_data_sample:
            if eval_item["input"] == train_item:
                contamination.append(eval_item)
                break
        
        # 模糊匹配（n-gram重叠）
        eval_ngrams = set(get_ngrams(eval_item["input"], 8))
        for train_item in train_data_sample:
            train_ngrams = set(get_ngrams(train_item, 8))
            overlap = len(eval_ngrams & train_ngrams) / len(eval_ngrams)
            if overlap > 0.8:
                contamination.append(eval_item)
                break
    
    return {
        "contaminated_count": len(contamination),
        "contamination_rate": len(contamination) / len(eval_data),
        "contaminated_items": contamination
    }

class EvalDatasetVersion:
    def __init__(self, name, version, data):
        self.name = name
        self.version = version
        self.data = data
        self.created_at = datetime.now()
        self.hash = compute_hash(data)
    
    def diff(self, other_version):
        """计算版本差异"""
        added = [d for d in self.data if d not in other_version.data]
        removed = [d for d in other_version.data if d not in self.data]
        return {"added": added, "removed": removed}

# eval_dataset_card.yaml
name: "客服对话评估集 v2.0"
version: "2.0"
created: "2026-06-01"
size: 2000
description: "电商客服对话质量评估数据集"

coverage:
  query_types:
    咨询: 500
    投诉: 400
    退款: 350
    技术支持: 350
    其他: 400

difficulty:
  简单: 600
  中等: 900
  困难: 500

quality:
  annotation_consistency: 0.87
  diversity_score: 0.82
  bias_check: "通过"
  contamination_check: "无污染"

limitation:
  - "仅覆盖电商领域"
  - "中文数据为主"
  - "不含多轮对话"