admin
5ab4602535
- 接入 SAM2 视频传播能力：新增 /api/ai/propagate，支持用当前帧 mask/polygon/bbox 作为 seed，通过 SAM2 video predictor 向前、向后或双向传播，并可保存为真实 annotation。 - 接入 SAM3 video tracker：通过独立 Python 3.12 external worker 调用 SAM3 video predictor/tracker，使用本地 checkpoint 与 bbox seed 执行视频级跟踪，并在模型状态中标记 video_track 能力。 - 完善 SAM 模型分发：sam_registry 按 model_id 明确区分 sam2 propagation 与 sam3 video_track，避免两个模型链路混用。 - 打通前端“传播片段”：VideoWorkspace 使用当前选中 mask 和当前 AI 模型调用后端传播接口，传播结果回写并刷新工作区已保存标注。 - 增强 SAM3 本地 checkpoint 配置：新增 sam3_checkpoint_path 配置和 .env.example 示例，状态检查改为基于本地 checkpoint/独立环境/模型包可用性。 - 完善视频拆帧参数：/api/media/parse 支持 parse_fps、max_frames、target_width，后端任务保存帧时间戳、源帧号和 frame_sequence 元数据。 - 增加运行时 schema 兼容处理：启动时为旧 frames 表补充 timestamp_ms 和 source_frame_number 列，避免旧库升级后缺字段。 - 强化 Canvas 标注编辑：补齐多边形闭合、点工具、顶点拖拽、边中点插入、Delete/Backspace 删除、区域合并和重叠去除等交互。 - 增强语义分类联动：选中 mask 后可通过右侧语义分类树更新标签、颜色和 class metadata，并同步到保存/导出链路。 - 增加关键帧时间轴体验：FrameTimeline 显示具体时间信息，并支持键盘左右方向键切换关键帧。 - 完善 AI 交互分割参数：前端保留正向点、反向点、框选和 interactive prompt 的调用状态，支持 SAM2 细化候选区域与 SAM3 bbox 入口。 - 扩展后端/前端 API 类型：新增 propagateMasks、传播请求/响应 schema，并补齐 annotation、导出、模型状态和任务接口的测试覆盖。 - 更新项目文档：同步 README、AGENTS、接口契约、需求冻结、设计冻结、前端元素审计、实施计划和测试计划，标明真实功能边界与剩余风险。 - 增加测试覆盖：补充 SAM2/SAM3 传播、SAM3 状态、媒体拆帧参数、Canvas 编辑、语义标签切换、时间轴、工作区传播和 API 合约测试。 - 加强仓库安全边界：将 sam3权重/ 加入 .gitignore，避免本地模型权重被误提交。 验证：npm run test:run；pytest backend/tests；npm run lint；npm run build；python -m py_compile；git diff --check。
111 lines
3.9 KiB
Python
111 lines
3.9 KiB
Python
"""Model registry for SAM runtimes and GPU status."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from config import settings
|
|
from services.sam2_engine import TORCH_AVAILABLE, sam_engine as sam2_engine
|
|
from services.sam3_engine import sam3_engine
|
|
|
|
try:
|
|
import torch
|
|
except Exception: # noqa: BLE001
|
|
torch = None # type: ignore[assignment]
|
|
|
|
|
|
class ModelUnavailableError(RuntimeError):
|
|
"""Raised when a selected model cannot run in this environment."""
|
|
|
|
|
|
class SAMRegistry:
|
|
"""Dispatch predictions to the selected SAM backend."""
|
|
|
|
def __init__(self) -> None:
|
|
self._engines = {
|
|
"sam2": sam2_engine,
|
|
"sam3": sam3_engine,
|
|
}
|
|
|
|
def normalize_model_id(self, model_id: str | None) -> str:
|
|
selected = (model_id or settings.sam_default_model or "sam2").lower()
|
|
if selected not in self._engines:
|
|
raise ValueError(f"Unsupported model: {model_id}")
|
|
return selected
|
|
|
|
def runtime_status(self, selected_model: str | None = None) -> dict[str, Any]:
|
|
return {
|
|
"selected_model": self.normalize_model_id(selected_model),
|
|
"gpu": self.gpu_status(),
|
|
"models": [engine.status() for engine in self._engines.values()],
|
|
}
|
|
|
|
def gpu_status(self) -> dict[str, Any]:
|
|
cuda_available = bool(TORCH_AVAILABLE and torch is not None and torch.cuda.is_available())
|
|
return {
|
|
"available": cuda_available,
|
|
"device": "cuda" if cuda_available else "cpu",
|
|
"name": torch.cuda.get_device_name(0) if cuda_available else None,
|
|
"torch_available": bool(TORCH_AVAILABLE),
|
|
"torch_version": getattr(torch, "__version__", None) if torch is not None else None,
|
|
"cuda_version": getattr(torch.version, "cuda", None) if torch is not None else None,
|
|
}
|
|
|
|
def _engine(self, model_id: str | None) -> Any:
|
|
return self._engines[self.normalize_model_id(model_id)]
|
|
|
|
def _ensure_available(self, model_id: str | None) -> Any:
|
|
engine = self._engine(model_id)
|
|
status = engine.status()
|
|
if not status["available"]:
|
|
raise ModelUnavailableError(status["message"])
|
|
return engine
|
|
|
|
def predict_points(self, model_id: str | None, image: Any, points: list[list[float]], labels: list[int]):
|
|
return self._ensure_available(model_id).predict_points(image, points, labels)
|
|
|
|
def predict_box(self, model_id: str | None, image: Any, box: list[float]):
|
|
return self._ensure_available(model_id).predict_box(image, box)
|
|
|
|
def predict_interactive(
|
|
self,
|
|
model_id: str | None,
|
|
image: Any,
|
|
box: list[float] | None,
|
|
points: list[list[float]],
|
|
labels: list[int],
|
|
):
|
|
model = self.normalize_model_id(model_id)
|
|
if model != "sam2":
|
|
raise NotImplementedError("Interactive box + point refinement is currently supported by SAM 2.")
|
|
return self._ensure_available(model).predict_interactive(image, box, points, labels)
|
|
|
|
def predict_auto(self, model_id: str | None, image: Any):
|
|
return self._ensure_available(model_id).predict_auto(image)
|
|
|
|
def predict_semantic(self, model_id: str | None, image: Any, text: str):
|
|
model = self.normalize_model_id(model_id)
|
|
if model == "sam3":
|
|
return self._ensure_available(model).predict_semantic(image, text)
|
|
return self._ensure_available(model).predict_auto(image)
|
|
|
|
def propagate_video(
|
|
self,
|
|
model_id: str | None,
|
|
frame_paths: list[str],
|
|
source_frame_index: int,
|
|
seed: dict[str, Any],
|
|
direction: str,
|
|
max_frames: int | None,
|
|
):
|
|
return self._ensure_available(model_id).propagate_video(
|
|
frame_paths,
|
|
source_frame_index,
|
|
seed,
|
|
direction=direction,
|
|
max_frames=max_frames,
|
|
)
|
|
|
|
|
|
sam_registry = SAMRegistry()
|