feat: 完善视频传播、标注编辑和拆帧闭环
- 接入 SAM2 视频传播能力：新增 /api/ai/propagate，支持用当前帧 mask/polygon/bbox 作为 seed，通过 SAM2 video predictor 向前、向后或双向传播，并可保存为真实 annotation。 - 接入 SAM3 video tracker：通过独立 Python 3.12 external worker 调用 SAM3 video predictor/tracker，使用本地 checkpoint 与 bbox seed 执行视频级跟踪，并在模型状态中标记 video_track 能力。 - 完善 SAM 模型分发：sam_registry 按 model_id 明确区分 sam2 propagation 与 sam3 video_track，避免两个模型链路混用。 - 打通前端“传播片段”：VideoWorkspace 使用当前选中 mask 和当前 AI 模型调用后端传播接口，传播结果回写并刷新工作区已保存标注。 - 增强 SAM3 本地 checkpoint 配置：新增 sam3_checkpoint_path 配置和 .env.example 示例，状态检查改为基于本地 checkpoint/独立环境/模型包可用性。 - 完善视频拆帧参数：/api/media/parse 支持 parse_fps、max_frames、target_width，后端任务保存帧时间戳、源帧号和 frame_sequence 元数据。 - 增加运行时 schema 兼容处理：启动时为旧 frames 表补充 timestamp_ms 和 source_frame_number 列，避免旧库升级后缺字段。 - 强化 Canvas 标注编辑：补齐多边形闭合、点工具、顶点拖拽、边中点插入、Delete/Backspace 删除、区域合并和重叠去除等交互。 - 增强语义分类联动：选中 mask 后可通过右侧语义分类树更新标签、颜色和 class metadata，并同步到保存/导出链路。 - 增加关键帧时间轴体验：FrameTimeline 显示具体时间信息，并支持键盘左右方向键切换关键帧。 - 完善 AI 交互分割参数：前端保留正向点、反向点、框选和 interactive prompt 的调用状态，支持 SAM2 细化候选区域与 SAM3 bbox 入口。 - 扩展后端/前端 API 类型：新增 propagateMasks、传播请求/响应 schema，并补齐 annotation、导出、模型状态和任务接口的测试覆盖。 - 更新项目文档：同步 README、AGENTS、接口契约、需求冻结、设计冻结、前端元素审计、实施计划和测试计划，标明真实功能边界与剩余风险。 - 增加测试覆盖：补充 SAM2/SAM3 传播、SAM3 状态、媒体拆帧参数、Canvas 编辑、语义标签切换、时间轴、工作区传播和 API 合约测试。 - 加强仓库安全边界：将 sam3权重/ 加入 .gitignore，避免本地模型权重被误提交。 验证：npm run test:run；pytest backend/tests；npm run lint；npm run build；python -m py_compile；git diff --check。
This commit is contained in:
@@ -4,6 +4,7 @@ from pathlib import Path
|
||||
import numpy as np
|
||||
|
||||
from services.sam3_engine import SAM3Engine
|
||||
from services.sam3_external_worker import _to_numpy
|
||||
|
||||
|
||||
class _Completed:
|
||||
@@ -14,6 +15,8 @@ class _Completed:
|
||||
|
||||
|
||||
def _external_settings(monkeypatch, python_path: Path):
|
||||
checkpoint_path = python_path.with_name("sam3.pt")
|
||||
checkpoint_path.write_bytes(b"checkpoint")
|
||||
python_path.write_text("#!/usr/bin/env python\n", encoding="utf-8")
|
||||
python_path.chmod(0o755)
|
||||
monkeypatch.setattr("services.sam3_engine.SAM3_PACKAGE_AVAILABLE", False)
|
||||
@@ -23,6 +26,7 @@ def _external_settings(monkeypatch, python_path: Path):
|
||||
monkeypatch.setattr("services.sam3_engine.settings.sam3_timeout_seconds", 10)
|
||||
monkeypatch.setattr("services.sam3_engine.settings.sam3_status_cache_seconds", 30)
|
||||
monkeypatch.setattr("services.sam3_engine.settings.sam3_confidence_threshold", 0.4)
|
||||
monkeypatch.setattr("services.sam3_engine.settings.sam3_checkpoint_path", str(checkpoint_path))
|
||||
|
||||
|
||||
def test_sam3_status_reports_external_runtime_ready(tmp_path, monkeypatch):
|
||||
@@ -30,9 +34,12 @@ def test_sam3_status_reports_external_runtime_ready(tmp_path, monkeypatch):
|
||||
|
||||
def fake_run(args, **_kwargs):
|
||||
assert "--status" in args
|
||||
assert _kwargs["env"]["SAM3_CHECKPOINT_PATH"].endswith("sam3.pt")
|
||||
return _Completed(stdout=json.dumps({
|
||||
"available": True,
|
||||
"package_available": True,
|
||||
"checkpoint_access": True,
|
||||
"checkpoint_path": _kwargs["env"]["SAM3_CHECKPOINT_PATH"],
|
||||
"python_ok": True,
|
||||
"torch_ok": True,
|
||||
"cuda_available": True,
|
||||
@@ -48,7 +55,10 @@ def test_sam3_status_reports_external_runtime_ready(tmp_path, monkeypatch):
|
||||
assert status["external_available"] is True
|
||||
assert status["package_available"] is True
|
||||
assert status["python_ok"] is True
|
||||
assert status["message"] == "SAM 3 external runtime is ready; model will load in the helper process on inference."
|
||||
assert status["checkpoint_exists"] is True
|
||||
assert status["checkpoint_path"].endswith("sam3.pt")
|
||||
assert status["supports"] == ["semantic", "box", "video_track"]
|
||||
assert status["message"] == "SAM 3 external runtime is ready; local checkpoint will load in the helper process on inference."
|
||||
|
||||
|
||||
def test_sam3_predict_semantic_uses_external_worker(tmp_path, monkeypatch):
|
||||
@@ -61,6 +71,7 @@ def test_sam3_predict_semantic_uses_external_worker(tmp_path, monkeypatch):
|
||||
return _Completed(stdout=json.dumps({
|
||||
"available": True,
|
||||
"package_available": True,
|
||||
"checkpoint_access": True,
|
||||
"python_ok": True,
|
||||
"torch_ok": True,
|
||||
"cuda_available": True,
|
||||
@@ -71,6 +82,7 @@ def test_sam3_predict_semantic_uses_external_worker(tmp_path, monkeypatch):
|
||||
request = json.loads(request_path.read_text(encoding="utf-8"))
|
||||
assert request["text"] == "vessel"
|
||||
assert request["confidence_threshold"] == 0.4
|
||||
assert request["checkpoint_path"].endswith("sam3.pt")
|
||||
assert Path(request["image_path"]).exists()
|
||||
return _Completed(stdout=json.dumps({
|
||||
"polygons": [[[0.1, 0.1], [0.9, 0.1], [0.9, 0.9]]],
|
||||
@@ -86,6 +98,97 @@ def test_sam3_predict_semantic_uses_external_worker(tmp_path, monkeypatch):
|
||||
assert any("--request" in args for args in calls)
|
||||
|
||||
|
||||
def test_sam3_predict_box_uses_external_worker(tmp_path, monkeypatch):
|
||||
_external_settings(monkeypatch, tmp_path / "python")
|
||||
|
||||
def fake_run(args, **_kwargs):
|
||||
if "--status" in args:
|
||||
return _Completed(stdout=json.dumps({
|
||||
"available": True,
|
||||
"package_available": True,
|
||||
"checkpoint_access": True,
|
||||
"python_ok": True,
|
||||
"torch_ok": True,
|
||||
"cuda_available": True,
|
||||
"device": "cuda",
|
||||
"message": "ready",
|
||||
}))
|
||||
request_path = Path(args[-1])
|
||||
request = json.loads(request_path.read_text(encoding="utf-8"))
|
||||
assert request["prompt_type"] == "box"
|
||||
assert request["box"] == [0.1, 0.2, 0.7, 0.8]
|
||||
assert request["text"] == ""
|
||||
return _Completed(stdout=json.dumps({
|
||||
"polygons": [[[0.1, 0.2], [0.7, 0.2], [0.7, 0.8]]],
|
||||
"scores": [0.88],
|
||||
}))
|
||||
|
||||
monkeypatch.setattr("services.sam3_engine.subprocess.run", fake_run)
|
||||
|
||||
polygons, scores = SAM3Engine().predict_box(
|
||||
np.zeros((8, 8, 3), dtype=np.uint8),
|
||||
[0.1, 0.2, 0.7, 0.8],
|
||||
)
|
||||
|
||||
assert polygons == [[[0.1, 0.2], [0.7, 0.2], [0.7, 0.8]]]
|
||||
assert scores == [0.88]
|
||||
|
||||
|
||||
def test_sam3_propagate_video_uses_external_worker(tmp_path, monkeypatch):
|
||||
_external_settings(monkeypatch, tmp_path / "python")
|
||||
frame_dir = tmp_path / "frames"
|
||||
frame_dir.mkdir()
|
||||
frame_paths = []
|
||||
for index in range(2):
|
||||
frame_path = frame_dir / f"frame_{index:06d}.jpg"
|
||||
frame_path.write_bytes(b"jpeg")
|
||||
frame_paths.append(str(frame_path))
|
||||
|
||||
def fake_run(args, **_kwargs):
|
||||
if "--status" in args:
|
||||
return _Completed(stdout=json.dumps({
|
||||
"available": True,
|
||||
"package_available": True,
|
||||
"checkpoint_access": True,
|
||||
"python_ok": True,
|
||||
"torch_ok": True,
|
||||
"cuda_available": True,
|
||||
"device": "cuda",
|
||||
"message": "ready",
|
||||
}))
|
||||
request_path = Path(args[-1])
|
||||
request = json.loads(request_path.read_text(encoding="utf-8"))
|
||||
assert request["prompt_type"] == "video_track"
|
||||
assert request["frame_dir"] == str(frame_dir)
|
||||
assert request["source_frame_index"] == 0
|
||||
assert request["direction"] == "forward"
|
||||
assert request["max_frames"] == 2
|
||||
assert request["seed"]["bbox"] == [0.1, 0.1, 0.2, 0.2]
|
||||
return _Completed(stdout=json.dumps({
|
||||
"frames": [
|
||||
{
|
||||
"frame_index": 1,
|
||||
"polygons": [[[0.2, 0.2], [0.4, 0.2], [0.4, 0.4]]],
|
||||
"scores": [0.7],
|
||||
"object_ids": [1],
|
||||
}
|
||||
]
|
||||
}))
|
||||
|
||||
monkeypatch.setattr("services.sam3_engine.subprocess.run", fake_run)
|
||||
|
||||
frames = SAM3Engine().propagate_video(
|
||||
frame_paths,
|
||||
0,
|
||||
{"bbox": [0.1, 0.1, 0.2, 0.2]},
|
||||
direction="forward",
|
||||
max_frames=2,
|
||||
)
|
||||
|
||||
assert frames[0]["frame_index"] == 1
|
||||
assert frames[0]["scores"] == [0.7]
|
||||
|
||||
|
||||
def test_sam3_predict_semantic_reports_external_errors(tmp_path, monkeypatch):
|
||||
_external_settings(monkeypatch, tmp_path / "python")
|
||||
|
||||
@@ -94,6 +197,7 @@ def test_sam3_predict_semantic_reports_external_errors(tmp_path, monkeypatch):
|
||||
return _Completed(stdout=json.dumps({
|
||||
"available": True,
|
||||
"package_available": True,
|
||||
"checkpoint_access": True,
|
||||
"python_ok": True,
|
||||
"torch_ok": True,
|
||||
"cuda_available": True,
|
||||
@@ -110,3 +214,32 @@ def test_sam3_predict_semantic_reports_external_errors(tmp_path, monkeypatch):
|
||||
assert "HF access denied" in str(exc)
|
||||
else:
|
||||
raise AssertionError("Expected SAM 3 external inference failure.")
|
||||
|
||||
|
||||
def test_sam3_worker_casts_floating_tensors_before_numpy():
|
||||
class FakeTensor:
|
||||
def __init__(self):
|
||||
self.float_called = False
|
||||
|
||||
def detach(self):
|
||||
return self
|
||||
|
||||
def is_floating_point(self):
|
||||
return True
|
||||
|
||||
def float(self):
|
||||
self.float_called = True
|
||||
return self
|
||||
|
||||
def cpu(self):
|
||||
return self
|
||||
|
||||
def numpy(self):
|
||||
return np.array([1.0], dtype=np.float32)
|
||||
|
||||
tensor = FakeTensor()
|
||||
|
||||
result = _to_numpy(tensor)
|
||||
|
||||
assert tensor.float_called is True
|
||||
assert result.tolist() == [1.0]
|
||||
|
||||
Reference in New Issue
Block a user