admin/Pre_Seg_Server
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: 完善视频传播、标注编辑和拆帧闭环

Browse Source 
- 接入 SAM2 视频传播能力：新增 /api/ai/propagate，支持用当前帧 mask/polygon/bbox 作为 seed，通过 SAM2 video predictor 向前、向后或双向传播，并可保存为真实 annotation。
- 接入 SAM3 video tracker：通过独立 Python 3.12 external worker 调用 SAM3 video predictor/tracker，使用本地 checkpoint 与 bbox seed 执行视频级跟踪，并在模型状态中标记 video_track 能力。
- 完善 SAM 模型分发：sam_registry 按 model_id 明确区分 sam2 propagation 与 sam3 video_track，避免两个模型链路混用。
- 打通前端“传播片段”：VideoWorkspace 使用当前选中 mask 和当前 AI 模型调用后端传播接口，传播结果回写并刷新工作区已保存标注。
- 增强 SAM3 本地 checkpoint 配置：新增 sam3_checkpoint_path 配置和 .env.example 示例，状态检查改为基于本地 checkpoint/独立环境/模型包可用性。
- 完善视频拆帧参数：/api/media/parse 支持 parse_fps、max_frames、target_width，后端任务保存帧时间戳、源帧号和 frame_sequence 元数据。
- 增加运行时 schema 兼容处理：启动时为旧 frames 表补充 timestamp_ms 和 source_frame_number 列，避免旧库升级后缺字段。
- 强化 Canvas 标注编辑：补齐多边形闭合、点工具、顶点拖拽、边中点插入、Delete/Backspace 删除、区域合并和重叠去除等交互。
- 增强语义分类联动：选中 mask 后可通过右侧语义分类树更新标签、颜色和 class metadata，并同步到保存/导出链路。
- 增加关键帧时间轴体验：FrameTimeline 显示具体时间信息，并支持键盘左右方向键切换关键帧。
- 完善 AI 交互分割参数：前端保留正向点、反向点、框选和 interactive prompt 的调用状态，支持 SAM2 细化候选区域与 SAM3 bbox 入口。
- 扩展后端/前端 API 类型：新增 propagateMasks、传播请求/响应 schema，并补齐 annotation、导出、模型状态和任务接口的测试覆盖。
- 更新项目文档：同步 README、AGENTS、接口契约、需求冻结、设计冻结、前端元素审计、实施计划和测试计划，标明真实功能边界与剩余风险。
- 增加测试覆盖：补充 SAM2/SAM3 传播、SAM3 状态、媒体拆帧参数、Canvas 编辑、语义标签切换、时间轴、工作区传播和 API 合约测试。
- 加强仓库安全边界：将 sam3权重/ 加入 .gitignore，避免本地模型权重被误提交。

验证：npm run test:run；pytest backend/tests；npm run lint；npm run build；python -m py_compile；git diff --check。
This commit is contained in:
admin
2026-05-01 20:27:33 +08:00
parent 689a9ba283
commit 5ab4602535
43 changed files with 2722 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
backend/tests/test_ai.py
View File

@@ -116,6 +116,44 @@ def test_predict_box_and_semantic_fallback(client, monkeypatch):
assert semantic_response.json()["scores"] == [0.5]
def test_predict_interactive_combines_box_and_points(client, monkeypatch):
_, frame, _ = _create_project_and_frame(client)
calls = {}
monkeypatch.setattr("routers.ai._load_frame_image", lambda frame: np.zeros((10, 10, 3), dtype=np.uint8))
def fake_predict_interactive(model, image, box, points, labels):
calls["model"] = model
calls["box"] = box
calls["points"] = points
calls["labels"] = labels
return (
[[[0.2, 0.2], [0.8, 0.2], [0.8, 0.8]]],
[0.88],
)
monkeypatch.setattr("routers.ai.sam_registry.predict_interactive", fake_predict_interactive)
response = client.post("/api/ai/predict", json={
"image_id": frame["id"],
"prompt_type": "interactive",
"prompt_data": {
"box": [0.1, 0.1, 0.9, 0.9],
"points": [[0.5, 0.5], [0.2, 0.2]],
"labels": [1, 0],
},
"model": "sam2",
})
assert response.status_code == 200
assert response.json()["scores"] == [0.88]
assert calls == {
"model": "sam2",
"box": [0.1, 0.1, 0.9, 0.9],
"points": [[0.5, 0.5], [0.2, 0.2]],
"labels": [1, 0],
}
def test_model_status_reports_runtime(client, monkeypatch):
monkeypatch.setattr("routers.ai.sam_registry.runtime_status", lambda selected_model=None: {
"selected_model": selected_model or "sam2",
@@ -170,6 +208,80 @@ def test_model_status_reports_runtime(client, monkeypatch):
assert body["models"][1]["available"] is False
def test_propagate_saves_tracked_annotations(client, monkeypatch):
project = client.post("/api/projects", json={"name": "Video Project"}).json()
frames = [
client.post(f"/api/projects/{project['id']}/frames", json={
"project_id": project["id"],
"frame_index": idx,
"image_url": f"frames/{idx}.jpg",
"width": 640,
"height": 360,
}).json()
for idx in range(3)
]
calls = {}
monkeypatch.setattr("routers.ai.download_file", lambda object_name: b"jpeg")
def fake_propagate_video(model, frame_paths, source_frame_index, seed, direction, max_frames):
calls["model"] = model
calls["source_frame_index"] = source_frame_index
calls["seed"] = seed
calls["direction"] = direction
calls["max_frames"] = max_frames
calls["frame_count"] = len(frame_paths)
return [
{
"frame_index": 0,
"polygons": [[[0.1, 0.1], [0.2, 0.1], [0.2, 0.2]]],
"scores": [0.9],
"object_ids": [1],
},
{
"frame_index": 1,
"polygons": [[[0.15, 0.15], [0.25, 0.15], [0.25, 0.25]]],
"scores": [0.8],
"object_ids": [1],
},
]
monkeypatch.setattr("routers.ai.sam_registry.propagate_video", fake_propagate_video)
response = client.post("/api/ai/propagate", json={
"project_id": project["id"],
"frame_id": frames[0]["id"],
"model": "sam2",
"direction": "forward",
"max_frames": 2,
"include_source": False,
"seed": {
"polygons": [[[0.1, 0.1], [0.2, 0.1], [0.2, 0.2]]],
"bbox": [0.1, 0.1, 0.1, 0.1],
"label": "胆囊",
"color": "#ff0000",
"class_metadata": {"id": "c1", "name": "胆囊", "color": "#ff0000", "zIndex": 20},
"template_id": None,
},
})
assert response.status_code == 200
body = response.json()
assert body["created_annotation_count"] == 1
assert body["processed_frame_count"] == 2
assert calls["model"] == "sam2"
assert calls["source_frame_index"] == 0
assert calls["direction"] == "forward"
assert calls["frame_count"] == 2
saved = body["annotations"][0]
assert saved["frame_id"] == frames[1]["id"]
assert saved["mask_data"]["source"] == "sam2_propagation"
assert saved["mask_data"]["class"]["name"] == "胆囊"
assert saved["mask_data"]["score"] == 0.8
listing = client.get(f"/api/ai/annotations?project_id={project['id']}")
assert len(listing.json()) == 1
def test_predict_validation_errors(client, monkeypatch):
project, _, _ = _create_project_and_frame(client)

54
backend/tests/test_media.py
View File

@@ -84,6 +84,12 @@ def test_parse_media_queues_background_task(client, monkeypatch):
assert data["progress"] == 0
assert data["project_id"] == project["id"]
assert data["celery_task_id"] == "celery-1"
assert data["payload"] == {
"source_type": "video",
"parse_fps": 5.0,
"max_frames": None,
"target_width": 640,
}
assert queued == [data["id"]]
assert published == [data["id"]]
@@ -94,6 +100,35 @@ def test_parse_media_queues_background_task(client, monkeypatch):
assert project_detail["status"] == "parsing"
def test_parse_media_accepts_frame_sequence_options(client, monkeypatch):
project = client.post("/api/projects", json={
"name": "Parse Options",
"video_path": "uploads/1/clip.mp4",
"source_type": "video",
"parse_fps": 30,
}).json()
class FakeAsyncResult:
id = "celery-options"
monkeypatch.setattr("routers.media.parse_project_media.delay", lambda task_id: FakeAsyncResult())
monkeypatch.setattr("routers.media.publish_task_progress_event", lambda task: None)
response = client.post(
f"/api/media/parse?project_id={project['id']}&parse_fps=15&max_frames=120&target_width=960"
)
assert response.status_code == 202
data = response.json()
assert data["payload"] == {
"source_type": "video",
"parse_fps": 15.0,
"max_frames": 120,
"target_width": 960,
}
assert client.get(f"/api/projects/{project['id']}").json()["parse_fps"] == 15.0
def test_parse_task_runner_registers_frames(client, db_session, monkeypatch, tmp_path):
from models import ProcessingTask
from services.media_task_runner import run_parse_media_task
@@ -118,10 +153,14 @@ def test_parse_task_runner_registers_frames(client, db_session, monkeypatch, tmp
frame_file.write_bytes(b"fake image")
monkeypatch.setattr("services.media_task_runner.download_file", lambda object_name: b"video")
monkeypatch.setattr("services.media_task_runner.parse_video", lambda local_path, output_dir, fps: ([str(frame_file)], 25.0))
monkeypatch.setattr(
"services.media_task_runner.parse_video",
lambda local_path, output_dir, fps, max_frames=None, target_width=640: ([str(frame_file)], 25.0),
)
monkeypatch.setattr("services.media_task_runner.extract_thumbnail", lambda local_path, thumbnail_path: open(thumbnail_path, "wb").write(b"thumb"))
monkeypatch.setattr("services.media_task_runner.upload_file", lambda *args, **kwargs: None)
monkeypatch.setattr("services.media_task_runner.upload_frames_to_minio", lambda frame_files, project_id: [f"projects/{project_id}/frames/frame_000001.jpg"])
monkeypatch.setattr("routers.projects.get_presigned_url", lambda object_name, expires=3600: f"http://storage/{object_name}")
published = []
monkeypatch.setattr(
"services.media_task_runner.publish_task_progress_event",
@@ -131,6 +170,17 @@ def test_parse_task_runner_registers_frames(client, db_session, monkeypatch, tmp
result = run_parse_media_task(db_session, task.id)
assert result["frames_extracted"] == 1
assert result["frame_sequence"] == {
"original_fps": 25.0,
"parse_fps": 5.0,
"frame_count": 1,
"duration_ms": 0.0,
"target_width": 640,
"frame_width": None,
"frame_height": None,
"max_frames": None,
"object_prefix": f"projects/{project['id']}/frames",
}
db_session.refresh(task)
assert task.status == "success"
assert task.progress == 100
@@ -140,6 +190,8 @@ def test_parse_task_runner_registers_frames(client, db_session, monkeypatch, tmp
assert project_detail["status"] == "ready"
frames = client.get(f"/api/projects/{project['id']}/frames").json()
assert "frame_000001.jpg" in frames[0]["image_url"]
assert frames[0]["timestamp_ms"] == 0.0
assert frames[0]["source_frame_number"] == 0
def test_parse_task_runner_skips_already_cancelled_task(db_session):

135
backend/tests/test_sam3_engine.py
View File

@@ -4,6 +4,7 @@ from pathlib import Path
import numpy as np
from services.sam3_engine import SAM3Engine
from services.sam3_external_worker import _to_numpy
class _Completed:
@@ -14,6 +15,8 @@ class _Completed:
def _external_settings(monkeypatch, python_path: Path):
checkpoint_path = python_path.with_name("sam3.pt")
checkpoint_path.write_bytes(b"checkpoint")
python_path.write_text("#!/usr/bin/env python\n", encoding="utf-8")
python_path.chmod(0o755)
monkeypatch.setattr("services.sam3_engine.SAM3_PACKAGE_AVAILABLE", False)
@@ -23,6 +26,7 @@ def _external_settings(monkeypatch, python_path: Path):
monkeypatch.setattr("services.sam3_engine.settings.sam3_timeout_seconds", 10)
monkeypatch.setattr("services.sam3_engine.settings.sam3_status_cache_seconds", 30)
monkeypatch.setattr("services.sam3_engine.settings.sam3_confidence_threshold", 0.4)
monkeypatch.setattr("services.sam3_engine.settings.sam3_checkpoint_path", str(checkpoint_path))
def test_sam3_status_reports_external_runtime_ready(tmp_path, monkeypatch):
@@ -30,9 +34,12 @@ def test_sam3_status_reports_external_runtime_ready(tmp_path, monkeypatch):
def fake_run(args, **_kwargs):
assert "--status" in args
assert _kwargs["env"]["SAM3_CHECKPOINT_PATH"].endswith("sam3.pt")
return _Completed(stdout=json.dumps({
"available": True,
"package_available": True,
"checkpoint_access": True,
"checkpoint_path": _kwargs["env"]["SAM3_CHECKPOINT_PATH"],
"python_ok": True,
"torch_ok": True,
"cuda_available": True,
@@ -48,7 +55,10 @@ def test_sam3_status_reports_external_runtime_ready(tmp_path, monkeypatch):
assert status["external_available"] is True
assert status["package_available"] is True
assert status["python_ok"] is True
assert status["message"] == "SAM 3 external runtime is ready; model will load in the helper process on inference."
assert status["checkpoint_exists"] is True
assert status["checkpoint_path"].endswith("sam3.pt")
assert status["supports"] == ["semantic", "box", "video_track"]
assert status["message"] == "SAM 3 external runtime is ready; local checkpoint will load in the helper process on inference."
def test_sam3_predict_semantic_uses_external_worker(tmp_path, monkeypatch):
@@ -61,6 +71,7 @@ def test_sam3_predict_semantic_uses_external_worker(tmp_path, monkeypatch):
return _Completed(stdout=json.dumps({
"available": True,
"package_available": True,
"checkpoint_access": True,
"python_ok": True,
"torch_ok": True,
"cuda_available": True,
@@ -71,6 +82,7 @@ def test_sam3_predict_semantic_uses_external_worker(tmp_path, monkeypatch):
request = json.loads(request_path.read_text(encoding="utf-8"))
assert request["text"] == "vessel"
assert request["confidence_threshold"] == 0.4
assert request["checkpoint_path"].endswith("sam3.pt")
assert Path(request["image_path"]).exists()
return _Completed(stdout=json.dumps({
"polygons": [[[0.1, 0.1], [0.9, 0.1], [0.9, 0.9]]],
@@ -86,6 +98,97 @@ def test_sam3_predict_semantic_uses_external_worker(tmp_path, monkeypatch):
assert any("--request" in args for args in calls)
def test_sam3_predict_box_uses_external_worker(tmp_path, monkeypatch):
_external_settings(monkeypatch, tmp_path / "python")
def fake_run(args, **_kwargs):
if "--status" in args:
return _Completed(stdout=json.dumps({
"available": True,
"package_available": True,
"checkpoint_access": True,
"python_ok": True,
"torch_ok": True,
"cuda_available": True,
"device": "cuda",
"message": "ready",
}))
request_path = Path(args[-1])
request = json.loads(request_path.read_text(encoding="utf-8"))
assert request["prompt_type"] == "box"
assert request["box"] == [0.1, 0.2, 0.7, 0.8]
assert request["text"] == ""
return _Completed(stdout=json.dumps({
"polygons": [[[0.1, 0.2], [0.7, 0.2], [0.7, 0.8]]],
"scores": [0.88],
}))
monkeypatch.setattr("services.sam3_engine.subprocess.run", fake_run)
polygons, scores = SAM3Engine().predict_box(
np.zeros((8, 8, 3), dtype=np.uint8),
[0.1, 0.2, 0.7, 0.8],
)
assert polygons == [[[0.1, 0.2], [0.7, 0.2], [0.7, 0.8]]]
assert scores == [0.88]
def test_sam3_propagate_video_uses_external_worker(tmp_path, monkeypatch):
_external_settings(monkeypatch, tmp_path / "python")
frame_dir = tmp_path / "frames"
frame_dir.mkdir()
frame_paths = []
for index in range(2):
frame_path = frame_dir / f"frame_{index:06d}.jpg"
frame_path.write_bytes(b"jpeg")
frame_paths.append(str(frame_path))
def fake_run(args, **_kwargs):
if "--status" in args:
return _Completed(stdout=json.dumps({
"available": True,
"package_available": True,
"checkpoint_access": True,
"python_ok": True,
"torch_ok": True,
"cuda_available": True,
"device": "cuda",
"message": "ready",
}))
request_path = Path(args[-1])
request = json.loads(request_path.read_text(encoding="utf-8"))
assert request["prompt_type"] == "video_track"
assert request["frame_dir"] == str(frame_dir)
assert request["source_frame_index"] == 0
assert request["direction"] == "forward"
assert request["max_frames"] == 2
assert request["seed"]["bbox"] == [0.1, 0.1, 0.2, 0.2]
return _Completed(stdout=json.dumps({
"frames": [
{
"frame_index": 1,
"polygons": [[[0.2, 0.2], [0.4, 0.2], [0.4, 0.4]]],
"scores": [0.7],
"object_ids": [1],
}
]
}))
monkeypatch.setattr("services.sam3_engine.subprocess.run", fake_run)
frames = SAM3Engine().propagate_video(
frame_paths,
0,
{"bbox": [0.1, 0.1, 0.2, 0.2]},
direction="forward",
max_frames=2,
)
assert frames[0]["frame_index"] == 1
assert frames[0]["scores"] == [0.7]
def test_sam3_predict_semantic_reports_external_errors(tmp_path, monkeypatch):
_external_settings(monkeypatch, tmp_path / "python")
@@ -94,6 +197,7 @@ def test_sam3_predict_semantic_reports_external_errors(tmp_path, monkeypatch):
return _Completed(stdout=json.dumps({
"available": True,
"package_available": True,
"checkpoint_access": True,
"python_ok": True,
"torch_ok": True,
"cuda_available": True,
@@ -110,3 +214,32 @@ def test_sam3_predict_semantic_reports_external_errors(tmp_path, monkeypatch):
assert "HF access denied" in str(exc)
else:
raise AssertionError("Expected SAM 3 external inference failure.")
def test_sam3_worker_casts_floating_tensors_before_numpy():
class FakeTensor:
def __init__(self):
self.float_called = False
def detach(self):
return self
def is_floating_point(self):
return True
def float(self):
self.float_called = True
return self
def cpu(self):
return self
def numpy(self):
return np.array([1.0], dtype=np.float32)
tensor = FakeTensor()
result = _to_numpy(tensor)
assert tensor.float_called is True
assert result.tolist() == [1.0]