多模态评估的特殊性 单模态（纯文本）评估已经相当成熟，但当我们进入多模态领域——图像、视频、音频与文本的交叉理解——评估的复杂度呈指数级增长。多模态模型不仅要理解每种模态的信息，还要在不同模态间建立语义关联。
多模态评估的核心挑战：
对齐问题：文本描述和图像内容是否语义一致？ 细粒度理解：模型是否真正"看到"了图像中的关键细节？ 跨模态推理：能否基于图像信息进行文本推理，或反向操作？ 评估成本：人工标注图文对的成本远高于纯文本 评估维度全景 多模态评估维度 ├── 视觉感知 │ ├── 图像识别（VQA, Image Captioning） │ ├── 细粒度理解（OCR, 属性识别） │ └── 空间推理（位置关系, 3D 理解） ├── 跨模态推理 │ ├── 图文推理（图→文推理） │ ├── 文图推理（文→图检索/生成） │ └── 多模态链式推理 ├── 多模态对话 │ ├── 多轮图像对话 │ └── 视频问答 └── 生成质量 ├── 图文一致性 ├── 视觉质量 └── 创意与忠实度 一、视觉理解评估 图像问答（VQA）评估 VQA 是最基础的多模态评估形式：给模型一张图片和一个问题，要求输出答案。
class VQAEvaluator: """VQA 评估器""" def __init__(self, eval_mode: str = "vqa_accuracy"): self.eval_mode = eval_mode def evaluate(self, predictions: list[dict]) -> dict: """ predictions: [{"question_id": int, "answer": str, "gt_answers": [str, ...]}] """ if self.eval_mode == "vqa_accuracy": return self._vqa_accuracy(predictions) elif self.eval_mode == "exact_match": return self._exact_match(predictions) def _vqa_accuracy(self, predictions: list[dict]) -> dict: """ 标准 VQA 准确率： 对每个问题，如果至少 3/10 的标注者给出了相同答案，则算正确 简化版：答案出现在 GT 答案列表中即算正确 """ correct = 0 for pred in predictions: gt = [a.strip().lower() for a in pred["gt_answers"]] ans = pred["answer"].strip().lower() # VQA 标准的 soft accuracy count = gt.count(ans) min_count = 1 # 简化：至少1个匹配 if count >= min_count: correct += min(1, count / 3.0) accuracy = correct / len(predictions) if predictions else 0 return {"vqa_accuracy": accuracy, "total": len(predictions)} def _exact_match(self, predictions: list[dict]) -> dict: correct = 0 for pred in predictions: gt = [a.strip().lower() for a in pred["gt_answers"]] ans = pred["answer"].strip().lower() if ans in gt: correct += 1 return {"exact_match": correct / len(predictions) if predictions else 0} class FineGrainedVQAEvaluator(VQAEvaluator): """细粒度 VQA 评估：按问题类型分桶""" QUESTION_TYPES = { "object": "图中有什么物体？", "count": "图中有几个XX？", "color": "XX是什么颜色的？", "spatial": "XX在YY的哪个位置？", "attribute": "XX有什么特征？", "relation": "XX和YY是什么关系？", "scene": "这是什么场景？", "ocr": "图中的文字写了什么？", } def evaluate_by_type(self, predictions: list[dict]) -> dict: from collections import defaultdict by_type = defaultdict(list) for pred in predictions: by_type[pred.get("question_type", "unknown")].append(pred) results = {} for q_type, preds in by_type.items(): results[q_type] = { "count": len(preds), "accuracy": self._exact_match(preds)["exact_match"], } return results 图像描述（Captioning）评估 class CaptioningEvaluator: """图像描述评估""" def __init__(self): self.metrics = {} def evaluate(self, prediction: str, references: list[str]) -> dict: results = {} # CIDEr-D: 专为图像描述设计的指标 results["cider"] = self._cider(prediction, references) # BLEU-4 results["bleu4"] = self._bleu(prediction, references, n=4) # METEOR results["meteor"] = self._meteor(prediction, references) # ROUGE-L results["rouge_l"] = self._rouge_l(prediction, references) # CLIPScore: 基于 CLIP 的图文匹配度 results["clip_score"] = self._clip_score(prediction, references) return results def _cider(self, pred: str, refs: list[str]) -> float: """CIDEr: 共识评估，基于 TF-IDF 加权的 n-gram 重叠""" # 简化实现，实际使用 pycocoevalcap from pycocoevalcap.cider.cider import Cider cider_scorer = Cider() gts = {0: refs} res = {0: [pred]} score, _ = cider_scorer.compute_score(gts, res) return score def _bleu(self, pred: str, refs: list[str], n: int = 4) -> float: from pycocoevalcap.bleu.bleu import Bleu bleu_scorer = Bleu(n) gts = {0: refs} res = {0: [pred]} score, _ = bleu_scorer.compute_score(gts, res) return score[n-1] def _meteor(self, pred: str, refs: list[str]) -> float: from pycocoevalcap.meteor.meteor import Meteor meteor_scorer = Meteor() gts = {0: refs} res = {0: [pred]} score, _ = meteor_scorer.compute_score(gts, res) return score def _rouge_l(self, pred: str, refs: list[str]) -> float: from pycocoevalcap.rouge.rouge import Rouge rouge_scorer = Rouge() gts = {0: refs} res = {0: [pred]} score, _ = rouge_scorer.compute_score(gts, res) return score def _clip_score(self, pred: str, refs: list[str], image_path: str = None) -> float: """CLIPScore: 使用 CLIP 计算图文匹配度""" from transformers import CLIPProcessor, CLIPModel from PIL import Image import torch model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32") processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") image = Image.open(image_path) inputs = processor(text=[pred], images=image, return_tensors="pt", padding=True) with torch.no_grad(): outputs = model(**inputs) # 归一化的图文相似度 score = outputs.logits_per_image.item() # 归一化到 0-1 return min(max(score / 100, 0), 1) OCR 评估 class OCREvaluator: """OCR 能力评估""" def evaluate(self, prediction: str, ground_truth: str) -> dict: results = {} # 字符级准确率 results["char_accuracy"] = self._char_accuracy(prediction, ground_truth) # 词级准确率 results["word_accuracy"] = self._word_accuracy(prediction, ground_truth) # 编辑距离 results["edit_distance"] = self._edit_distance(prediction, ground_truth) # 归一化编辑距离 max_len = max(len(prediction), len(ground_truth), 1) results["normalized_edit_distance"] = results["edit_distance"] / max_len # ANLS (Average Normalized Levenshtein Similarity) results["anls"] = 1 - results["normalized_edit_distance"] return results def _char_accuracy(self, pred: str, gt: str) -> float: """字符级准确率""" if not gt: return 1.0 if not pred else 0.0 correct = sum(1 for p, g in zip(pred, gt) if p == g) # 加上长度差异惩罚 correct += 0 # 多出或缺少的字符算错 return correct / len(gt) def _word_accuracy(self, pred: str, gt: str) -> float: pred_words = pred.split() gt_words = gt.split() if not gt_words: return 1.0 if not pred_words else 0.0 correct = sum(1 for p, g in zip(pred_words, gt_words) if p == g) return correct / len(gt_words) def _edit_distance(self, s1: str, s2: str) -> int: """Levenshtein 编辑距离""" if len(s1) < len(s2): return self._edit_distance(s2, s1) if len(s2) == 0: return len(s1) previous_row = range(len(s2) + 1) for i, c1 in enumerate(s1): current_row = [i + 1] for j, c2 in enumerate(s2): insertions = previous_row[j + 1] + 1 deletions = current_row[j] + 1 substitutions = previous_row[j] + (c1 != c2) current_row.append(min(insertions, deletions, substitutions)) previous_row = current_row return previous_row[-1] 二、跨模态推理评估 图文推理任务 class CrossModalReasoningEvaluator: """跨模态推理评估""" TASK_TYPES = [ "visual_entailment", # 视觉蕴含：图→文 是否支持 "visual_reasoning", # 视觉推理：基于图的逻辑推理 "image_text_matching", # 图文匹配 "visual_commonsense", # 视觉常识推理 "multimodal_cot", # 多模态链式推理 ] def evaluate_visual_entailment(self, predictions: list[dict]) -> dict: """ 视觉蕴含：判断文本假设是否被图像支持 标签: entailment / neutral / contradiction """ from sklearn.metrics import classification_report, accuracy_score labels = ["entailment", "neutral", "contradiction"] y_true = [p["gt_label"] for p in predictions] y_pred = [p["pred_label"] for p in predictions] report = classification_report(y_true, y_pred, labels=labels, output_dict=True) return { "accuracy": accuracy_score(y_true, y_pred), "per_class": {l: report[l] for l in labels}, } def evaluate_image_text_matching(self, predictions: list[dict]) -> dict: """ 图文匹配：给定图片和多个文本，选择最匹配的 """ correct = 0 for pred in predictions: if pred["pred_match"] == pred["gt_match"]: correct += 1 return {"accuracy": correct / len(predictions) if predictions else 0} def evaluate_multimodal_cot(self, predictions: list[dict]) -> dict: """ 多模态链式推理：评估推理步骤和最终答案 """ results = [] for pred in predictions: # 评估最终答案 answer_correct = self._check_answer(pred["pred_answer"], pred["gt_answer"]) # 评估推理步骤（使用 LLM-as-Judge） reasoning_score = self._evaluate_reasoning( pred["image_description"], pred["reasoning_steps"], pred["pred_answer"] ) results.append({ "answer_correct": answer_correct, "reasoning_score": reasoning_score, }) return { "answer_accuracy": sum(r["answer_correct"] for r in results) / len(results), "avg_reasoning_score": sum(r["reasoning_score"] for r in results) / len(results), } def _check_answer(self, pred: str, gt: str) -> bool: pred_clean = pred.strip().lower() gt_clean = gt.strip().lower() return gt_clean in pred_clean or pred_clean == gt_clean def _evaluate_reasoning(self, image_desc: str, reasoning: str, answer: str) -> float: """使用 LLM 评估推理质量""" prompt = f"""请评估以下多模态推理的质量（0-10分）： 图像描述：{image_desc} 推理过程：{reasoning} 最终答案：{answer} 评估维度： - 推理是否基于图像信息 - 逻辑是否连贯 - 是否有跳步或错误 请输出一个数字（0-10）。""" # 调用 LLM 评估 score = call_llm("gpt-4o", prompt) try: return float(score.strip()) / 10.0 except ValueError: return 0.5 主流多模态基准对比 基准 评估能力 任务数 模态 特点 VQAv2 视觉问答 1.1M 图+文 经典 VQA 基准 GQA 场景图推理 22M 图+文 结构化推理 MMBench 综合多模态 4K+ 图+文 多维能力评估 MMMU 学科多模态 11.5K 图+文 大学级别学科 MathVista 数学视觉推理 6K+ 图+文 数学+视觉 MMMU-Health 医学多模态 1K+ 图+文 医学领域 VideoMME 视频理解 900 视频+文 长视频理解 SEED-Bench 多场景理解 19K 图/视频+文 多模态多场景 三、图文一致性评估 生成图像的文本一致性 当模型从文本生成图像（或反向）时，需要评估跨模态的一致性：
...
