2026 本地 AI 技术栈：从模型到应用的完整方案

┌─────────────────────────────────────────────────┐
│              Application Layer                  │
│   Agent Framework / RAG / Workflow Engine       │
├─────────────────────────────────────────────────┤
│              Frontend Layer                     │
│   Open WebUI / LobeChat / LibreChat            │
├─────────────────────────────────────────────────┤
│              API Gateway Layer                  │
│   OpenAI Compatible API / Load Balancer         │
├─────────────────────────────────────────────────┤
│              Inference Layer                    │
│   vLLM / SGLang / Ollama / TGI                 │
├─────────────────────────────────────────────────┤
│              Model Layer                        │
│   Llama / Qwen / DeepSeek / Mistral            │
└─────────────────────────────────────────────────┘

# 用 Nginx 做多推理引擎负载均衡
upstream llm_backend {
    least_conn;
    server vllm-1:8000 max_fails=3 fail_timeout=30s;
    server vllm-2:8000 max_fails=3 fail_timeout=30s;
    server sglang-1:30000 backup;
}

server {
    listen 443 ssl http2;
    server_name llm.internal.example.com;

    ssl_certificate /etc/ssl/certs/llm.crt;
    ssl_certificate_key /etc/ssl/private/llm.key;

    # API 认证
    auth_request /auth;

    location / {
        proxy_pass http://llm_backend;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_buffering off;  # 流式输出必须关闭缓冲
        proxy_read_timeout 300s;
    }

    location /auth {
        internal;
        proxy_pass http://auth-service:5000/verify;
        proxy_pass_request_body off;
        proxy_set_header Content-Length "";
        proxy_set_header X-Original-URI $request_uri;
    }
}

# 推理性能估算公式
tokens_per_second = min(
    gpu_memory_bandwidth / model_size_bytes,  # 内存带宽瓶颈
    gpu_compute_tflops * efficiency / ops_per_token  # 计算瓶颈
)

# 示例：Llama 3.1 70B Q4 on A100 80G
# model_size = 40 GB
# gpu_bandwidth = 2000 GB/s
# theoretical_max = 2000 / 40 = 50 tokens/s (单请求)
# 实际效率约 70% → ~35 tokens/s

# docker-compose.full-stack.yml
version: "3.9"

services:
  # ========== 推理层 ==========
  vllm:
    image: vllm/vllm-openai:latest
    container_name: vllm
    runtime: nvidia
    environment:
      - HF_TOKEN=${HF_TOKEN}
    volumes:
      - model_cache:/root/.cache/huggingface
    ports:
      - "8000:8000"
    command:
      - --model=meta-llama/Llama-3.1-70B-Instruct
      - --tensor-parallel-size=4
      - --gpu-memory-utilization=0.90
      - --max-model-len=32768
      - --enable-prefix-caching
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 10
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 4
              capabilities: [gpu]

  vllm-embed:
    image: vllm/vllm-openai:latest
    container_name: vllm-embed
    runtime: nvidia
    volumes:
      - model_cache:/root/.cache/huggingface
    ports:
      - "8001:8000"
    command:
      - --model=nomic-ai/nomic-embed-text-v1.5
      - --gpu-memory-utilization=0.5
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]

  # ========== 前端层 ==========
  open-webui:
    image: ghcr.io/open-webui/open-webui:main
    container_name: open-webui
    ports:
      - "3000:8080"
    volumes:
      - webui_data:/app/backend/data
    environment:
      - OLLAMA_BASE_URL=http://ollama:11434
      - OPENAI_API_BASE_URL=http://vllm:8000/v1
      - OPENAI_API_KEY=none
      - WEBUI_AUTH=true
      - ENABLE_SIGNUP=false
      - DATABASE_URL=postgresql://webui:${DB_PASSWORD}@postgres:5432/webui
    depends_on:
      - vllm
      - postgres

  ollama:
    image: ollama/ollama:latest
    container_name: ollama
    ports:
      - "11434:11434"
    volumes:
      - ollama_data:/root/.ollama
    environment:
      - OLLAMA_KEEP_ALIVE=24h

  # ========== 应用层 ==========
  qdrant:
    image: qdrant/qdrant:latest
    container_name: qdrant
    ports:
      - "6333:6333"
    volumes:
      - qdrant_data:/qdrant/storage

  dify:
    image: langgenius/dify-api:latest
    container_name: dify
    ports:
      - "5001:5001"
    environment:
      - DB_HOST=postgres
      - DB_PASSWORD=${DB_PASSWORD}
      - REDIS_HOST=redis
      - VECTOR_STORE=qdrant
      - QDRANT_URL=http://qdrant:6333
    depends_on:
      - postgres
      - redis
      - qdrant

  # ========== 基础设施 ==========
  postgres:
    image: postgres:16-alpine
    container_name: postgres
    environment:
      - POSTGRES_PASSWORD=${DB_PASSWORD}
      - POSTGRES_DB=webui
    volumes:
      - postgres_data:/var/lib/postgresql/data

  redis:
    image: redis:7-alpine
    container_name: redis
    volumes:
      - redis_data:/data

  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus_data:/prometheus
    ports:
      - "9090:9090"

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD}
    volumes:
      - grafana_data:/var/lib/grafana
    ports:
      - "3001:3000"

  dcgm-exporter:
    image: nvcr.io/nvidia/k8s/dcgm-exporter:latest
    container_name: dcgm-exporter
    runtime: nvidia
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    ports:
      - "9400:9400"

volumes:
  model_cache:
  ollama_data:
  webui_data:
  postgres_data:
  redis_data:
  qdrant_data:
  prometheus_data:
  grafana_data:

# docker-compose.network.yml 片段
networks:
  frontend:
    driver: bridge
    internal: false  # 对外暴露
  backend:
    driver: bridge
    internal: true   # 仅内部访问
  inference:
    driver: bridge
    internal: true   # 推理引擎隔离

services:
  open-webui:
    networks: [frontend, backend]
  vllm:
    networks: [backend, inference]
  postgres:
    networks: [backend]

# API Key 认证中间件
from fastapi import Request, HTTPException
import hashlib, hmac, time

VALID_API_KEYS = set()  # 从数据库加载

async def verify_api_key(request: Request):
    api_key = request.headers.get("Authorization", "").replace("Bearer ", "")
    if not api_key or api_key not in VALID_API_KEYS:
        raise HTTPException(status_code=401, detail="Invalid API key")

    # 速率限制
    client_ip = request.client.host
    rate_key = f"rate:{client_ip}:{api_key[:8]}"
    # ... Redis 速率限制

# 输入过滤 + 输出审核
class SafetyFilter:
    BLOCKED_PATTERNS = [
        r"ignore (previous|above) instructions",
        r"system prompt",
        r"<\|.*\|>",  # 特殊 token 注入
    ]

    def check_input(self, text: str) -> bool:
        import re
        for pattern in self.BLOCKED_PATTERNS:
            if re.search(pattern, text, re.IGNORECASE):
                return False
        return True

    def check_output(self, text: str) -> str:
        # PII 脱敏
        import re
        text = re.sub(r'\b\d{16,19}\b', '[REDACTED]', text)  # 信用卡
        text = re.sub(r'\b\d{3}-\d{2}-\d{4}\b', '[SSN]', text)  # SSN
        return text

#!/bin/bash
# model-update.sh
set -euo pipefail

MODEL_NAME="meta-llama/Llama-3.1-70B-Instruct"
BACKUP_DIR="/data/models/backup"

# 1. 备份当前模型
cp -r /data/models/current $BACKUP_DIR/$(date +%Y%m%d)

# 2. 拉取新版本
huggingface-cli download $MODEL_NAME --local-dir /data/models/new

# 3. 评估测试
python eval.py --model /data/models/new --benchmark mmlu,gsm8k,humaneval

# 4. 金丝雀部署
docker run --name vllm-canary -d \
  vllm/vllm-openai:latest \
  --model /data/models/new \
  --port 8001

# 5. 流量切换（通过 Nginx weight 调整）
# 6. 清理旧版本（保留最近 3 个）
ls -dt $BACKUP_DIR/*/ | tail -n +4 | xargs rm -rf

# 每日备份
0 2 * * * docker exec postgres pg_dump -U webui webui | gzip > /backup/db_$(date +\%Y\%m\%d).sql.gz
0 3 * * * rsync -avz /data/models/ backup-server:/backup/models/
0 4 * * * docker exec qdrant qdrant-cli snapshot create

# 保留策略：7 天日备 + 4 周周备 + 12 月月备