引言

传统软件测试的基石是确定性：相同输入产生相同输出。LLM Agent 打破了这个前提，让 CI/CD 测试面临根本性挑战。2026年，随着 Agent 评估框架的成熟，一套可落地的 CI/CD 测试方法论终于成型。本文将带你构建完整的 Agent 测试金字塔。

一、Agent 测试金字塔

                    ┌─────────┐
                    │  E2E    │  ← 端到端场景测试（5-10个）
                   ┌┴─────────┴┐
                  │  Integration │  ← 多 Agent 协作测试（20-50个）
                 ┌┴──────────────┴┐
                │   Evaluation     │  ← LLM 评判测试（50-100个）
               ┌┴──────────────────┴┐
              │     Component         │  ← 工具/Prompt 测试（200+）
             ┌┴──────────────────────┴┐
            │       Unit Test           │  ← 纯函数测试（500+）
            └───────────────────────────┘

二、第一层：单元测试（确定性层）

单元测试只测试不涉及 LLM 的部分：数据处理、工具执行的解析逻辑、Prompt 模板渲染。

import pytest

class TestPromptTemplate:
    """Prompt 模板渲染测试"""
    
    def test_system_prompt_renders_correctly(self):
        template = SystemPromptTemplate(
            role="research_assistant",
            tools=["search", "calculator"],
            constraints=["cite sources", "be concise"]
        )
        rendered = template.render()
        
        assert "research_assistant" in rendered
        assert "search" in rendered
        assert "calculator" in rendered
        assert "cite sources" in rendered
    
    def test_few_shot_template_with_examples(self):
        template = FewShotTemplate(
            system="You are a classifier",
            examples=[
                {"input": "I love it", "output": "positive"},
                {"input": "Terrible", "output": "negative"},
            ],
            query="{user_input}"
        )
        rendered = template.render(user_input="Amazing!")
        
        assert "positive" in rendered
        assert "negative" in rendered
        assert "Amazing!" in rendered


class TestToolParsing:
    """工具调用解析测试"""
    
    @pytest.mark.parametrize("raw_output,expected_tool,expected_args", [
        ('{"tool": "search", "args": {"q": "weather"}}', "search", {"q": "weather"}),
        ('```json\n{"tool": "calc", "args": {"expr": "1+1"}}\n```', "calc", {"expr": "1+1"}),
        ('I\'ll use the search tool: {"tool": "search", "args": {"q": "news"}}', "search", {"q": "news"}),
    ])
    def test_parse_tool_call(self, raw_output, expected_tool, expected_args):
        result = parse_tool_call(raw_output)
        assert result.tool == expected_tool
        assert result.args == expected_args
    
    def test_parse_malformed_output(self):
        with pytest.raises(ToolParseError):
            parse_tool_call("This is not JSON at all")

三、第二层：组件测试（Mock LLM）

使用 Mock LLM 测试 Agent 的控制流，确保工作流逻辑正确。

from unittest.mock import AsyncMock, patch

class TestAgentWorkflow:
    """Agent 工作流逻辑测试（不调用真实 LLM）"""
    
    @pytest.fixture
    def mock_llm(self):
        llm = AsyncMock()
        # 预设 LLM 返回序列
        llm.side_effect = [
            MockResponse(content="I'll search for that", tool_calls=[
                ToolCall(tool="search", args={"q": "test query"})
            ]),
            MockResponse(content="Based on the results...", tool_calls=[]),
        ]
        return llm
    
    async def test_agent_calls_search_then_responds(self, mock_llm):
        agent = Agent(llm=mock_llm, tools=[search_tool])
        result = await agent.run("What is the latest news?")
        
        # 验证调用序列
        assert mock_llm.call_count == 2
        assert mock_llm.call_args_list[0].args[0].messages[-1].content == "What is the latest news?"
        
        # 验证工具被调用
        assert "search" in agent.execution_trace[0].tool_calls[0].tool
    
    async def test_agent_respects_max_iterations(self):
        """测试 Agent 不会无限循环"""
        mock_llm = AsyncMock()
        mock_llm.return_value = MockResponse(
            content="Let me search again",
            tool_calls=[ToolCall(tool="search", args={"q": "test"})]
        )
        
        agent = Agent(llm=mock_llm, tools=[search_tool], max_iterations=5)
        result = await agent.run("Infinite loop test")
        
        assert mock_llm.call_count == 5  # 恰好 5 次
        assert result.status == "max_iterations_reached"
    
    async def test_agent_handles_tool_error_gracefully(self):
        """测试工具出错时 Agent 能优雅处理"""
        mock_llm = AsyncMock()
        mock_llm.side_effect = [
            MockResponse(content="", tool_calls=[
                ToolCall(tool="failing_tool", args={})
            ]),
            MockResponse(content="I encountered an error, let me try another approach", tool_calls=[]),
        ]
        
        agent = Agent(llm=mock_llm, tools=[failing_tool])
        result = await agent.run("Test error handling")
        
        assert result.status == "success"
        assert len(agent.execution_trace) == 2

四、第三层：LLM 评估测试

使用 LLM-as-Judge 对 Agent 输出进行质量评估。这是 Agent 测试的核心层。

from pydantic import BaseModel

class EvaluationResult(BaseModel):
    score: float          # 0-1
    reasoning: str
    passed: bool

class LLMJudge:
    """LLM 评判器"""
    
    def __init__(self, judge_model: str = "gpt-5"):
        self.model = judge_model
        self.client = OpenAI()
    
    async def evaluate(
        self,
        task: str,
        output: str,
        criteria: list[str],
        reference: str | None = None
    ) -> EvaluationResult:
        prompt = f"""
        Task: {task}
        
        Output to evaluate:
        {output}
        
        Criteria:
        {chr(10).join(f'- {c}' for c in criteria)}
        
        {"Reference answer:" + reference if reference else ""}
        
        Score from 0.0 to 1.0. Be strict.
        Respond in JSON: {{"score": float, "reasoning": str, "passed": bool}}
        """
        
        response = self.client.chat.completions.create(
            model=self.model,
            messages=[{"role": "user", "content": prompt}],
            response_format={"type": "json_object"},
            temperature=0.0  # 评判需要确定性
        )
        
        return EvaluationResult(**json.loads(response.choices[0].message.content))


# 测试用例定义
TEST_CASES = [
    {
        "id": "research_001",
        "task": "Research the latest developments in quantum computing",
        "criteria": [
            "Response mentions at least 2 recent breakthroughs (2025-2026)",
            "Response includes credible sources",
            "Response is under 500 words",
            "Response does not hallucinate facts",
        ],
        "min_score": 0.8,
    },
    {
        "id": "code_001", 
        "task": "Write a Python function to detect cycle in linked list",
        "criteria": [
            "Code is syntactically correct",
            "Algorithm has O(n) time complexity",
            "Code handles edge cases (empty list, single node)",
            "Code includes type hints",
        ],
        "min_score": 0.85,
    },
]

@pytest.mark.parametrize("case", TEST_CASES)
@pytest.mark.asyncio
async def test_agent_quality(case, agent, judge):
    """LLM 评判的 Agent 质量测试"""
    output = await agent.run(case["task"])
    result = await judge.evaluate(
        task=case["task"],
        output=output,
        criteria=case["criteria"]
    )
    
    assert result.score >= case["min_score"], \
        f"Score {result.score} < {case['min_score']}. Reason: {result.reasoning}"

五、第四层：集成测试

测试多 Agent 协作场景。

class TestMultiAgentIntegration:
    """多 Agent 集成测试"""
    
    async def test_research_writing_pipeline(self):
        """研究 Agent + 写作 Agent 的协作"""
        researcher = Agent(
            name="researcher",
            llm=real_llm,  # 使用真实 LLM
            tools=[web_search, paper_search]
        )
        writer = Agent(
            name="writer", 
            llm=real_llm,
            tools=[grammar_check]
        )
        
        pipeline = Pipeline([researcher, writer])
        result = await pipeline.run("Write a blog about AGI in 2026")
        
        # 结构性验证
        assert len(result) > 500
        assert len(result) < 3000
        assert "AGI" in result
        assert "2026" in result
        
        # 质量验证
        eval = await judge.evaluate(
            task="Write a blog about AGI in 2026",
            output=result,
            criteria=["well-structured", "factual", "engaging"]
        )
        assert eval.score > 0.75

六、CI/CD Pipeline 集成

GitHub Actions 配置

# .github/workflows/agent-tests.yml
name: Agent Test Suite

on:
  pull_request:
    paths:
      - "agents/**"
      - "prompts/**"
      - "tools/**"
  schedule:
    - cron: "0 6 * * *"  # 每天定时全量测试

jobs:
  unit-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
      - run: pip install -e ".[test]"
      - run: pytest tests/unit/ -v --cov=agents --cov-report=xml
    
  component-tests:
    runs-on: ubuntu-latest
    needs: unit-tests
    steps:
      - uses: actions/checkout@v4
      - run: pytest tests/component/ -v
  
  llm-eval-tests:
    runs-on: ubuntu-latest
    needs: component-tests
    if: github.event_name == 'pull_request'
    strategy:
      matrix:
        test_group: ["research", "coding", "reasoning", "safety"]
    steps:
      - uses: actions/checkout@v4
      - run: pytest tests/evaluation/ -v --group=${{ matrix.test_group }}
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      - name: Upload eval results
        uses: actions/upload-artifact@v4
        with:
          name: eval-results-${{ matrix.test_group }}
          path: test-results/
  
  regression-check:
    runs-on: ubuntu-latest
    needs: llm-eval-tests
    steps:
      - name: Compare with baseline
        run: |
          python scripts/eval_regression.py \
            --current test-results/ \
            --baseline eval-baselines/$(git merge-base HEAD main)/ \
            --threshold 0.05

回归检测脚本

class EvalRegression:
    """评估回归检测"""
    
    def compare(
        self,
        current: dict[str, float],
        baseline: dict[str, float],
        threshold: float = 0.05
    ) -> bool:
        """如果任何指标下降超过 threshold，返回 True（存在回归）"""
        regressions = []
        for test_id, score in current.items():
            baseline_score = baseline.get(test_id)
            if baseline_score is None:
                continue
            if score < baseline_score - threshold:
                regressions.append({
                    "test": test_id,
                    "baseline": baseline_score,
                    "current": score,
                    "drop": baseline_score - score
                })
        
        if regressions:
            print("⚠️  Evaluation regression detected:")
            for r in regressions:
                print(f"  {r['test']}: {r['baseline']:.2f} → {r['current']:.2f} (↓{r['drop']:.2f})")
            return True
        return False

七、测试数据管理

class TestDataset:
    """测试数据集管理"""
    
    def __init__(self, version: str = "v1.0"):
        self.version = version
        self.cases = self._load()
    
    def _load(self) -> list[TestCase]:
        """从版本化的测试数据文件加载"""
        path = f"testdata/{self.version}/cases.jsonl"
        cases = []
        with open(path) as f:
            for line in f:
                data = json.loads(line)
                cases.append(TestCase(**data))
        return cases
    
    def split(self, ratio: float = 0.8):
        """分割为开发和回归集"""
        shuffled = sorted(self.cases, key=lambda x: hash(x.id))
        split_at = int(len(shuffled) * ratio)
        return shuffled[:split_at], shuffled[split_at:]

八、测试成本控制

LLM 测试的最大挑战是成本。推荐策略：

策略节省比例实现方式
分层执行70%PR 只跑单元+组件层，合并后跑评估层
模型降级85%评估测试用 GPT-4o-mini 代替 GPT-5
采样测试60%从 100 个用例中随机抽 30 个
缓存 Mock90%缓存 LLM 响应，重复测试用缓存
并行化0%（提速）评估用例并行执行

结语

Agent 评估自动化不是一次性工程，而是持续演进的体系。从确定性单元测试到 LLM-as-Judge 评估，每一层都在补充其他层的盲区。关键原则是：能确定测试的就不要用 LLM 评判，能用便宜模型评判的就不要用昂贵模型。让每一分测试预算都花在真正需要的地方。

加入讨论

这篇文章有姊妹讨论帖在硅基AGI论坛 — 全球首个碳基硅基认知交流平台。

碳基与硅基的智慧碰撞，认知差异创造无限可能。