语义缓存

LLM缓存的价值 LLM推理是昂贵的——每次调用消耗GPU算力和API费用。但很多请求是重复或高度相似的。缓存可以让"已经算过的不再重算"，是投入产出比最高的优化手段。 三层缓存架构 请求 → L1: 精确缓存 → L2: 语义缓存 → L3: 前缀缓存 → LLM L1：精确缓存 import redis.asyncio as redis class ExactCache: """精确匹配缓存：相同输入返回相同输出""" def __init__(self, redis_url="redis://localhost:6379"): self.redis = redis.from_url(redis_url) def cache_key(self, model, messages, temperature, max_tokens): """生成缓存键""" content = json.dumps({ "model": model, "messages": messages, "temperature": temperature, "max_tokens": max_tokens, }, sort_keys=True, ensure_ascii=False) return hashlib.sha256(content.encode()).hexdigest() async def get(self, model, messages, **params): key = self.cache_key(model, messages, params.get("temperature", 0.7), params.get("max_tokens", 2048)) cached = await self.redis.get(key) return json.loads(cached) if cached else None async def set(self, model, messages, response, ttl=3600, **params): key = self.cache_key(model, messages, params.get("temperature", 0.7), params.get("max_tokens", 2048)) await self.redis.setex(key, ttl, json.dumps(response, ensure_ascii=False)) 适用场景 temperature=0的确定性输出 常见问题FAQ 系统提示词相同的请求 L2：语义缓存 class SemanticCache: """语义缓存：相似查询命中缓存""" def __init__(self, vector_store, embed_model, threshold=0.95): self.store = vector_store self.embed_model = embed_model self.threshold = threshold async def get(self, query, model="default", **params): # 向量化查询 query_embedding = await self.embed_model.embed(query) # 搜索相似缓存 results = await self.store.search( query_embedding, filter={"model": model}, top_k=1 ) if results and results[0]["score"] >= self.threshold: cached = json.loads(results[0]["document"]) # 检查参数兼容性 if self.params_compatible(cached["params"], params): return cached["response"] return None async def set(self, query, response, model="default", ttl=3600, **params): embedding = await self.embed_model.embed(query) await self.store.add( id=str(uuid.uuid4()), embedding=embedding, document=json.dumps({ "query": query, "response": response, "model": model, "params": params, "timestamp": time.time(), }, ensure_ascii=False), ttl=ttl ) def params_compatible(self, cached_params, request_params): """检查缓存参数是否兼容当前请求""" # temperature差异较大则不兼容 if abs(cached_params.get("temperature", 0.7) - request_params.get("temperature", 0.7)) > 0.1: return False return True 语义缓存的阈值选择 阈值 命中率 准确率 适用场景 0.99 低 极高 严格场景（医疗、法律） 0.95 中 高 通用场景 0.90 高 中 FAQ类场景 0.85 很高 低 不推荐（风险大） L3：前缀缓存（KV Cache共享） class PrefixCacheManager: """前缀缓存：共享相同前缀的KV Cache""" def __init__(self): self.prefix_cache = {} # prefix_hash -> kv_cache def compute_prefix_hash(self, messages): """计算消息前缀的哈希""" # 系统提示 + 历史对话通常构成共享前缀 prefix_text = json.dumps(messages[:-1]) # 除最后一条消息 return hashlib.sha256(prefix_text.encode()).hexdigest() async def get_kv_cache(self, messages): """获取前缀的KV Cache""" prefix_hash = self.compute_prefix_hash(messages) return self.prefix_cache.get(prefix_hash) async def store_kv_cache(self, messages, kv_cache): """存储前缀的KV Cache""" prefix_hash = self.compute_prefix_hash(messages) self.prefix_cache[prefix_hash] = kv_cache vLLM内置前缀缓存支持： ...