AI 视频风格迁移：艺术风格一键应用

from runway import RunwayClient

client = RunwayClient(api_key="your_api_key")

# 上传源视频
source_video = client.upload("input_video.mp4")

# 风格迁移
styled_video = client.video_to_video(
    video_id=source_video.id,
    prompt="Van Gogh Starry Night style, thick brushstrokes, "
            "swirling sky, vibrant blues and yellows",
    style_strength=0.75,  # 0-1，风格强度
    preserve_structure=True,  # 保持结构
    duration=15,  # 输出时长（秒）
    resolution="1080p"
)

# 下载结果
styled_video.download("output_vangogh.mp4")

源视频：城市夜景延时摄影（10秒）
风格 prompt："Van Gogh Starry Night style, heavy impasto, "
            "swirling brushstrokes, starry sky, "
            "blue and gold palette"
参数：style_strength=0.7, preserve_structure=True
结果：城市灯光变成星空中闪烁的星星，建筑线条融入漩涡状笔触

源视频：现代都市街拍（15秒）
风格 prompt："Cyberpunk style, neon lights, "
            "futuristic city, holographic signs, "
            "purple and cyan lighting, rain-soaked streets"
参数：style_strength=0.8, preserve_structure=True
结果：街道出现霓虹灯和全息广告，色调变为赛博朋克标志性紫青色

源视频
    ↓
逐帧提取（24fps）
    ↓
每帧用 ControlNet 处理
    ↓
Canny 边缘检测（保持结构）
    ↓
Depth 深度图（保持空间）
    ↓
Stable Diffusion 风格生成
    ↓
帧序列 → 视频
    ↓
帧间稳定化（防止闪烁）
    ↓
成品视频

import torch
from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
import cv2
import numpy as np

class VideoStyleTransfer:
    """基于 ControlNet 的视频风格迁移"""
    
    def __init__(self, style_prompt, model="runwayml/stable-diffusion-v1-5"):
        # 加载 ControlNet
        self.controlnet = ControlNetModel.from_pretrained(
            "lllyasviel/sd-controlnet-canny"
        )
        
        # 加载管道
        self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
            model,
            controlnet=self.controlnet,
            torch_dtype=torch.float16
        ).to("cuda")
        
        self.style_prompt = style_prompt
        self.sparator = torch.Generator("cuda").manual_seed(42)
    
    def process_video(self, input_path, output_path, 
                     style_strength=0.7, max_frames=None):
        """处理视频"""
        cap = cv2.VideoCapture(input_path)
        fps = cap.get(cv2.CAP_PROP_FPS)
        width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
        height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
        
        # 输出视频
        out = cv2.VideoWriter(
            output_path,
            cv2.VideoWriter_fourcc(*'mp4v'),
            fps, (width, height)
        )
        
        frame_count = 0
        prev_frame = None
        
        while True:
            ret, frame = cap.read()
            if not ret:
                break
            
            if max_frames and frame_count >= max_frames:
                break
            
            # 1. Canny 边缘检测
            canny = self._canny_edge(frame)
            
            # 2. ControlNet 风格化
            styled = self.pipe(
                prompt=self.style_prompt,
                image=canny,
                controlnet_conditioning_scale=style_strength,
                num_inference_steps=20,
                generator=self.generator
            ).images[0]
            
            # 3. 转换为 OpenCV 格式
            styled_cv = cv2.cvtColor(np.array(styled), cv2.COLOR_RGB2BGR)
            styled_cv = cv2.resize(styled_cv, (width, height))
            
            # 4. 帧间稳定化（简单加权平均）
            if prev_frame is not None:
                styled_cv = cv2.addWeighted(
                    prev_frame, 0.3, styled_cv, 0.7, 0
                )
            
            # 写入
            out.write(styled_cv)
            prev_frame = styled_cv.copy()
            
            frame_count += 1
            print(f"处理进度：{frame_count} 帧")
        
        cap.release()
        out.release()
        print(f"完成！输出：{output_path}")
    
    def _canny_edge(self, frame):
        """Canny 边缘检测"""
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        edges = cv2.Canny(gray, 100, 200)
        edges_rgb = cv2.cvtColor(edges, cv2.COLOR_GRAY2RGB)
        return Image.fromarray(edges_rgb)

# 实例化
stylizer = VideoStyleTransfer(
    style_prompt="Ukiyo-e style, Japanese woodblock print, "
                 "flat colors, bold outlines, minimalist"
)

# 处理视频
stylizer.process_video(
    input_path="input.mp4",
    output_path="output_ukiyoe.mp4",
    style_strength=0.75,
    max_frames=72  # 只处理前 3 秒（24fps × 3s）
)

import torch
import torch.nn as nn
import torchvision.models as models
import torchvision.transforms as transforms

class RealTimeStyleTransfer(nn.Module):
    """实时风格迁移模型（轻量版）"""
    
    def __init__(self, style_image_path, model_size="small"):
        super().__init__()
        
        # 编码器（固定）
        if model_size == "small":
            self.encoder = models.mobilenet_v3_small(pretrained=True).features
        else:
            self.encoder = models.mobilenet_v3_large(pretrained=True).features
        
        # 风格特征提取
        self.style_features = self._extract_style_features(style_image_path)
        
        # 解码器（可训练）
        self.decoder = self._build_decoder()
        
        # AdaIN 参数
        self.adain_params = nn.ParameterDict({
            "style_mean": torch.zeros(1, 1, 1, 1),
            "style_std": torch.ones(1, 1, 1, 1)
        })
    
    def forward(self, x):
        # 编码
        content_features = self.encoder(x)
        
        # AdaIN
        normalized = self._instance_norm(content_features)
        styled = (normalized * self.adain_params["style_std"] + 
                  self.adain_params["style_mean"])
        
        # 解码
        output = self.decoder(styled)
        
        return output
    
    def _instance_norm(self, x):
        """实例归一化"""
        x_mean = x.mean(dim=[1, 2, 3], keepdim=True)
        x_std = x.std(dim=[1, 2, 3], keepdim=True)
        return (x - x_mean) / (x_std + 1e-8)

# 转换为 ONNX（用于移动端）
torch.onnx.export(
    model,
    dummy_input,
    "style_transfer.onnx",
    input_names=["input"],
    output_names=["output"],
    dynamic_axes={"input": {0: "batch"}, "output": {0: "batch"}}
)

# Android 部署（使用 ONNX Runtime）
// Kotlin 代码
val ortEnvironment = OrtEnvironment.getEnvironment()
val session = ortEnvironment.createSession("style_transfer.onnx")

// 处理摄像头帧
fun processFrame(bitmap: Bitmap): Bitmap {
    val inputTensor = bitmapToFloat32Tensor(bitmap)
    val output = session.run(mapOf("input" to inputTensor))
    return float32TensorToBitmap(output[0])
}

# 光流稳定化
def stabilize_with_optical_flow(prev_frame, curr_frame, styled_curr):
    # 计算光流
    flow = cv2.calcOpticalFlowFarneback(
        cv2.cvtColor(prev_frame, cv2.COLOR_BGR2GRAY),
        cv2.cvtColor(curr_frame, cv2.COLOR_BGR2GRAY),
        None, 0.5, 3, 15, 3, 5, 1.2, 0
    )
    
    # 根据光流扭曲前一帧
    h, w = prev_frame.shape[:2]
    remap_x = np.arange(w) + flow[:, :, 0]
    remap_y = np.arange(h)[:, np.newaxis] + flow[:, :, 1]
    
    prev_warped = cv2.remap(prev_frame, 
                              remap_x.astype(np.float32), 
                              remap_y.astype(np.float32), 
                              cv2.INTER_LINEAR)
    
    # 混合
    result = cv2.addWeighted(prev_warped, 0.3, styled_curr, 0.7, 0)
    return result