自愈

引言 Agent系统的运维复杂度远超传统Web应用——LLM推理服务的GPU故障、工具调用的外部依赖故障、Token消耗突增导致的成本爆炸，这些都需要自动化运维系统能够及时发现并自动处理。2026年，成熟的Agent系统已经实现了从"手动运维"到"自愈+自动扩缩容"的跨越，将人工干预频率降低了90%以上。 自动化运维架构 ┌──────────────────────────────────────────────────────────┐ │ Agent自动化运维体系 │ │ │ │ ┌────────────┐ ┌──────────┐ ┌──────────┐ │ │ │ 健康检测 │ │ 告警 │ │ 自愈 │ │ │ │ Health │ │ Alerts │ │ Self- │ │ │ │ Check │ │ │ │ healing │ │ │ └─────┬──────┘ └────┬─────┘ └────┬─────┘ │ │ │ │ │ │ │ ▼ ▼ ▼ │ │ ┌─────────────────────────────────────────────┐ │ │ │ 决策引擎 (Decision Engine) │ │ │ │ - 规则引擎 │ │ │ │ - ML预测模型 │ │ │ │ - 执行计划生成 │ │ │ └─────────────────┬───────────────────────────┘ │ │ │ │ │ ┌───────────┼───────────────┐ │ │ ▼ ▼ ▼ │ │ ┌─────────┐ ┌──────────┐ ┌────────────┐ │ │ │Auto- │ │Auto- │ │Auto- │ │ │ │Scaling │ │Remediation│ │Configuration│ │ │ │扩缩容 │ │修复 │ │配置 │ │ │ └─────────┘ └──────────┘ └────────────┘ │ └──────────────────────────────────────────────────────────┘ 自愈机制 健康检查 class HealthChecker: """Agent系统健康检查""" CHECKS = { "llm_service": { "endpoint": "http://llm-service:8080/health", "timeout": 5, "expected_status": 200, "degraded_threshold_ms": 500, }, "tool_services": { "endpoint": "http://tool-service:8080/health", "timeout": 3, "expected_status": 200, }, "vector_db": { "endpoint": "http://qdrant:6333/health", "timeout": 5, }, "redis": { "command": "PING", "expected_response": "PONG", } } async def run_all_checks(self) -> dict: """运行所有健康检查""" results = {} for name, check in self.CHECKS.items(): try: result = await self._run_check(name, check) results[name] = result except Exception as e: results[name] = { "status": "unhealthy", "error": str(e), "timestamp": datetime.now().isoformat() } # 计算整体健康分 healthy_count = sum( 1 for r in results.values() if r["status"] == "healthy" ) health_score = healthy_count / len(results) return { "overall_health": health_score, "status": "healthy" if health_score > 0.8 else "degraded", "checks": results, "timestamp": datetime.now().isoformat() } async def _run_check(self, name: str, check: dict) -> dict: """运行单项检查""" start = time.monotonic() if "endpoint" in check: async with httpx.AsyncClient(timeout=check["timeout"]) as client: response = await client.get(check["endpoint"]) latency_ms = (time.monotonic() - start) * 1000 if response.status_code != check["expected_status"]: return { "status": "unhealthy", "actual_status": response.status_code, "expected_status": check["expected_status"] } status = "healthy" if latency_ms > check.get("degraded_threshold_ms", 1000): status = "degraded" return { "status": status, "latency_ms": latency_ms, "status_code": response.status_code } 自愈动作 class SelfHealingActions: """自愈动作库""" async def restart_service(self, service_name: str) -> dict: """重启服务""" logger.warning(f"Self-healing: restarting {service_name}") try: # 通过K8s API重启 await self.k8s_client.restart_deployment(service_name) # 等待就绪 await self._wait_for_ready(service_name, timeout=120) return { "action": "restart_service", "service": service_name, "success": True } except Exception as e: logger.error(f"Self-healing failed for {service_name}: {e}") return { "action": "restart_service", "service": service_name, "success": False, "error": str(e) } async def clear_cache(self, cache_type: str) -> dict: """清理缓存""" if cache_type == "redis": await self.redis_client.flushdb() elif cache_type == "vector": await self.vector_db.clear_cache() return {"action": "clear_cache", "type": cache_type, "success": True} async def scale_service( self, service_name: str, replicas: int ) -> dict: """扩缩容服务""" current = await self.k8s_client.get_deployment_replicas(service_name) if current == replicas: return {"action": "scale", "changed": False} await self.k8s_client.scale_deployment(service_name, replicas) return { "action": "scale", "service": service_name, "from": current, "to": replicas } async def failover_to_backup(self, service_name: str) -> dict: """故障转移到备用服务""" backup_config = self.backup_configs.get(service_name) if not backup_config: raise ValueError(f"No backup configured for {service_name}") # 切换流量到备用 await self.traffic_router.switch_to_backup( service_name, backup_config["endpoint"] ) return { "action": "failover", "service": service_name, "backup": backup_config["name"] } 自动扩缩容 预测性扩缩容 class PredictiveAutoScaler: """预测性自动扩缩容""" def __init__(self, k8s_client, metrics_client): self.k8s = k8s_client self.metrics = metrics_client self.prediction_model = None # 加载训练好的预测模型 async def run_scaling_loop(self): """扩缩容主循环""" while True: try: # 1. 收集当前指标 current_metrics = await self._collect_metrics() # 2. 预测未来负载 predicted_load = await self._predict_load( current_metrics, horizon_minutes=15 ) # 3. 计算目标副本数 target_replicas = self._calculate_target_replicas( predicted_load ) # 4. 执行扩缩容 await self._apply_scaling(target_replicas) # 5. 等待下一次评估 await asyncio.sleep(60) # 每分钟评估一次 except Exception as e: logger.error(f"Auto-scaling error: {e}") await asyncio.sleep(60) async def _predict_load(self, current: dict, horizon_minutes: int) -> dict: """预测未来负载""" # 使用历史数据 + 当前趋势预测 if self.prediction_model: features = self._extract_prediction_features(current) prediction = await self.prediction_model.predict(features) return prediction else: # 简单线性回归预测 trend = self._calculate_trend(current) predicted_qps = current["qps"] * (1 + trend * horizon_minutes / 60) return {"predicted_qps": max(0, predicted_qps)} def _calculate_target_replicas(self, predicted_load: dict) -> dict: """计算目标副本数""" targets = {} for service, config in self.service_configs.items(): predicted_qps = predicted_load.get("predicted_qps", 0) # 每个副本能处理的QPS qps_per_replica = config.get("qps_per_replica", 10) # 目标副本数（加20%安全余量） target = int(predicted_qps / qps_per_replica * 1.2) # 应用上下限 target = max(config["min_replicas"], target) target = min(config["max_replicas"], target) targets[service] = target return targets 扩缩容策略 class ScalingStrategy: """扩缩容策略""" STRATEGIES = { "conservative": { "scale_up_step": 1, # 每次只加1个副本 "scale_down_step": 1, # 每次只减1个副本 "scale_up_cooldown": 120, # 扩容冷却2分钟 "scale_down_cooldown": 300, # 缩容冷却5分钟 }, "aggressive": { "scale_up_step": 5, "scale_down_step": 2, "scale_up_cooldown": 30, "scale_down_cooldown": 180, }, "predictive": { "lookahead_minutes": 15, "safety_margin": 0.3, # 30%安全余量 } } async def execute_scaling( self, service: str, current: int, target: int, strategy: str = "conservative" ): """执行扩缩容""" config = self.STRATEGIES[strategy] if target > current: # 扩容 step = config["scale_up_step"] new_replicas = min(current + step, target) logger.info( f"Scaling up {service}: {current} -> {new_replicas}" ) await self.k8s.scale_deployment(service, new_replicas) elif target < current: # 缩容——更保守 step = config["scale_down_step"] new_replicas = max(current - step, target) # 检查是否有正在处理的请求 active_requests = await self._get_active_requests(service) if active_requests > 0: logger.info( f"Postponing scale down for {service}: " f"{active_requests} active requests" ) return logger.info( f"Scaling down {service}: {current} -> {new_replicas}" ) await self.k8s.scale_deployment(service, new_replicas) 故障预测 class FailurePredictor: """故障预测器""" async def predict_failures(self) -> list: """预测可能发生的故障""" predictions = [] # 1. 基于指标的预测 metrics_anomalies = await self._detect_metric_anomalies() for anomaly in metrics_anomalies: predictions.append({ "type": "metric_anomaly", "service": anomaly["service"], "probability": anomaly["probability"], "description": anomaly["description"], "recommended_action": anomaly["action"] }) # 2. 基于日志的预测 log_patterns = await self._analyze_log_patterns() for pattern in log_patterns: if pattern["risk_score"] > 0.7: predictions.append({ "type": "log_pattern", "pattern": pattern["pattern"], "probability": pattern["risk_score"], "description": f"Detected pattern: {pattern['description']}" }) # 3. 基于依赖健康的预测 dependency_health = await self._check_dependency_health() for dep in dependency_health: if dep["health_score"] < 0.5: predictions.append({ "type": "dependency", "dependency": dep["name"], "probability": 1 - dep["health_score"], "description": f"Dependency {dep['name']} is unhealthy" }) return sorted(predictions, key=lambda p: p["probability"], reverse=True) AIOps实践 class AIOpsEngine: """AIOps引擎""" async def analyze_incident(self, incident: dict) -> dict: """AI辅助事故分析""" # 1. 收集相关日志/指标/Trace context = await self._gather_incident_context(incident) # 2. 使用LLM分析根因 analysis = await self.llm.analyze( prompt=f""" Analyze this incident and identify the root cause: Incident: {incident} Context: {context} Provide: 1. Most likely root cause 2. Contributing factors 3. Recommended actions 4. Prevention measures """ ) # 3. 从历史事故中找相似案例 similar_incidents = await self._find_similar_incidents(incident) return { "incident_id": incident["id"], "root_cause_analysis": analysis["root_cause"], "recommended_actions": analysis["actions"], "similar_incidents": similar_incidents, "confidence": analysis["confidence"] } async def generate_runbook(self, incident_type: str) -> str: """自动生成Runbook""" return await self.llm.generate( prompt=f""" Generate a detailed runbook for handling {incident_type} incidents. Include: 1. Detection steps 2. Diagnosis procedures 3. Resolution steps 4. Verification steps 5. Post-mortem template """ ) 总结 Agent自动化运维的核心目标是"让系统自己管理自己"。自愈机制通过健康检查+修复动作的组合，能够自动处理80%以上的常见故障。预测性扩缩容通过提前预判负载变化，避免了响应式扩缩容的滞后性。AIOps则通过AI辅助事故分析和Runbook生成，显著提升了排障效率。 ...