多模态评估指南：视觉语言模型怎么测？

         ┌──────────┬──────────┬──────────┐
         │  感知层   │  推理层   │  生成层   │
图像理解 │ 对象识别  │ 因果推理  │ 描述生成  │
视频理解 │ 动作识别  │ 时序推理  │ 摘要生成  │
3D 理解  │ 空间感知  │ 交互推理  │ 场景描述  │
         └──────────┴──────────┴──────────┘

├── 粗粒度感知 (对象存在/计数/场景分类)
├── 细粒度感知 (属性/空间关系/OCR)
├── 推理 (物理/社会/逻辑)
└── 关系 (图内/跨图)

# Yes/No 问答，避免开放式生成的评分问题
def evaluate_mme(model, dataset):
    correct = 0
    for item in dataset:
        for question, answer in item['questions']:
            pred = model.answer(item['image'], question)
            if normalize(pred) == answer.lower():
                correct += 1
    return correct / len(dataset)

def evaluate_video(model, video_path, questions):
    for q in questions:
        if q['type'] == 'temporal':
            frames = sample_frames(video_path, fps=2)    # 密集采帧
        elif q['type'] == 'action':
            frames = sample_keyframes(video_path, n=16)  # 关键帧
        else:
            frames = sample_uniform(video_path, n=8)     # 均匀采样
        answer = model.answer(frames, q['question'])

def check_contamination(train_data, test_data):
    from imagehash import phash
    train_hashes = {phash(img) for img in train_data}
    leaked = sum(1 for img in test_data if phash(img) in train_hashes)
    return leaked / len(test_data)

def test_vision_dependency(model, questions):
    with_image = [model.answer(q['image'], q['question']) for q in questions]
    without_image = [model.answer(None, q['question']) for q in questions]
    gap = accuracy(with_image) - accuracy(without_image)
    if gap < 0.1:
        print("警告：模型可能未有效利用视觉信息")