admin/Pre_Seg_Server
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: 建立 SAM2 标注闭环基线

Browse Source 
- 打通工作区真实标注闭环：支持手工多边形、矩形、圆形、点区域和线段生成 mask，并可保存、回显、更新和删除后端 annotation。

- 增强 polygon 编辑器：支持顶点拖动、顶点删除、边中点插入、多 polygon 子区域选择编辑，以及区域合并和区域去除。

- 接入 GT mask 导入：后端支持二值/多类别 mask 拆分、contour 转 polygon、distance transform seed point，前端支持导入、回显和 seed point 拖动编辑。

- 完善导出能力：COCO JSON 导出对齐前端，PNG mask ZIP 同时包含单标注 mask、按 zIndex 融合的 semantic_frame 和 semantic_classes.json。

- 打通异步任务管理：新增任务取消、重试、失败详情接口与 Dashboard 控件，worker 支持取消状态检查并通过 Redis/WebSocket 推送 cancelled 事件。

- 对接 Dashboard 后端数据：概览统计、解析队列和实时流转记录从 FastAPI 聚合接口与 WebSocket 更新。

- 增强 AI 推理参数：前端发送 crop_to_prompt、auto_filter_background 和 min_score，后端支持点/框 prompt 局部裁剪推理、结果回映射和负向点/低分过滤。

- 接入 SAM3 基础设施：新增独立 Python 3.12 sam3 环境安装脚本、外部 worker helper、后端桥接和真实 Python/CUDA/包/HF checkpoint access 状态检测。

- 保留 SAM3 授权边界：当前官方 facebook/sam3 gated 权重未授权时状态接口会返回不可用，不伪装成可推理。

- 增强前端状态管理：新增 mask undo/redo 历史栈、AI 模型选择状态、保存状态 dirty/draft/saved 流转和项目状态归一化。

- 更新前端 API 封装：补充 annotation CRUD、GT mask import、mask ZIP export、task cancel/retry/detail、AI runtime status 和 prediction options。

- 更新 UI 控件：ToolsPalette、AISegmentation、VideoWorkspace 和 CanvasArea 接入真实操作、导入导出、撤销重做、任务控制和模型状态。

- 新增 polygon-clipping 依赖，用于前端区域 union/difference 几何运算。

- 完善后端 schemas/status/progress：补充 AI 模型外部状态字段、任务 cancelled 状态和进度事件 payload。

- 补充测试覆盖：新增后端任务控制、SAM3 桥接、GT mask、导出融合、AI options 测试；补充前端 Canvas、Dashboard、VideoWorkspace、ToolsPalette、API 和 store 测试。

- 更新 README、AGENTS 和 doc 文档：冻结当前需求/设计/测试计划，标注真实功能、剩余 Mock、SAM3 授权边界和后续实施顺序。
This commit is contained in:
admin
2026-05-01 15:26:25 +08:00
parent f020ff3b4f
commit 689a9ba283
48 changed files with 3280 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
backend/tests/test_ai.py
View File

@@ -1,4 +1,5 @@
import numpy as np
import cv2
def _create_project_and_frame(client):
@@ -46,6 +47,46 @@ def test_predict_accepts_point_object_with_labels(client, monkeypatch):
assert calls["args"] == ([[0.5, 0.5], [0.1, 0.1]], [1, 0])
def test_predict_applies_crop_and_background_filter_options(client, monkeypatch):
_, frame, _ = _create_project_and_frame(client)
calls = {}
monkeypatch.setattr("routers.ai._load_frame_image", lambda frame: np.zeros((100, 200, 3), dtype=np.uint8))
def fake_predict_points(model, image, points, labels):
calls["shape"] = image.shape
calls["points"] = points
calls["labels"] = labels
return (
[
[[0.0, 0.0], [0.2, 0.0], [0.2, 0.2]],
[[0.45, 0.45], [0.55, 0.45], [0.55, 0.55]],
],
[0.9, 0.01],
)
monkeypatch.setattr("routers.ai.sam_registry.predict_points", fake_predict_points)
response = client.post("/api/ai/predict", json={
"image_id": frame["id"],
"prompt_type": "point",
"prompt_data": {"points": [[0.5, 0.5], [0.52, 0.52]], "labels": [1, 0]},
"options": {
"crop_to_prompt": True,
"crop_margin": 0.1,
"auto_filter_background": True,
"min_score": 0.05,
},
})
assert response.status_code == 200
assert calls["shape"][0] < 100
assert calls["shape"][1] < 200
assert calls["labels"] == [1, 0]
assert response.json()["scores"] == [0.9]
polygon = response.json()["polygons"][0]
assert all(0.0 <= coord <= 1.0 for point in polygon for coord in point)
def test_predict_box_and_semantic_fallback(client, monkeypatch):
_, frame, _ = _create_project_and_frame(client)
monkeypatch.setattr("routers.ai._load_frame_image", lambda frame: np.zeros((10, 10, 3), dtype=np.uint8))
@@ -246,3 +287,62 @@ def test_update_and_delete_annotation_validation(client):
f"/api/ai/annotations/{saved['id']}",
json={"template_id": 999},
).status_code == 404
def test_import_gt_mask_creates_annotations_with_seed_points(client):
project, frame, template = _create_project_and_frame(client)
mask = np.zeros((360, 640), dtype=np.uint8)
cv2.rectangle(mask, (100, 80), (260, 220), 255, thickness=-1)
ok, encoded = cv2.imencode(".png", mask)
assert ok
response = client.post(
"/api/ai/import-gt-mask",
data={
"project_id": str(project["id"]),
"frame_id": str(frame["id"]),
"template_id": str(template["id"]),
"label": "Imported GT",
"color": "#22c55e",
},
files={"file": ("mask.png", encoded.tobytes(), "image/png")},
)
assert response.status_code == 201
body = response.json()
assert len(body) == 1
assert body[0]["project_id"] == project["id"]
assert body[0]["frame_id"] == frame["id"]
assert body[0]["template_id"] == template["id"]
assert body[0]["mask_data"]["label"] == "Imported GT"
assert body[0]["mask_data"]["source"] == "gt_mask"
assert body[0]["mask_data"]["gt_label_value"] == 255
assert len(body[0]["mask_data"]["polygons"][0]) >= 3
assert len(body[0]["points"]) == 1
assert 0.0 <= body[0]["points"][0][0] <= 1.0
assert 0.0 <= body[0]["points"][0][1] <= 1.0
def test_import_gt_mask_splits_label_values(client):
project, frame, _ = _create_project_and_frame(client)
mask = np.zeros((360, 640), dtype=np.uint8)
cv2.rectangle(mask, (20, 20), (120, 120), 1, thickness=-1)
cv2.rectangle(mask, (220, 80), (320, 180), 2, thickness=-1)
ok, encoded = cv2.imencode(".png", mask)
assert ok
response = client.post(
"/api/ai/import-gt-mask",
data={
"project_id": str(project["id"]),
"frame_id": str(frame["id"]),
"label": "GT Class",
},
files={"file": ("labels.png", encoded.tobytes(), "image/png")},
)
assert response.status_code == 201
body = sorted(response.json(), key=lambda item: item["mask_data"]["gt_label_value"])
assert [item["mask_data"]["gt_label_value"] for item in body] == [1, 2]
assert [item["mask_data"]["label"] for item in body] == ["GT Class 1", "GT Class 2"]
assert all(len(item["points"]) == 1 for item in body)

2
backend/tests/test_dashboard.py
View File

@@ -59,7 +59,9 @@ def test_dashboard_overview_uses_persisted_records(client, db_session):
"name": "Pending Project",
"progress": 35,
"status": "正在使用 FFmpeg/OpenCV 拆帧",
"raw_status": "running",
"frame_count": 0,
"error": None,
"updated_at": body["tasks"][0]["updated_at"],
},
]

54
backend/tests/test_export.py
View File

@@ -1,6 +1,10 @@
import zipfile
import json
from io import BytesIO
import cv2
import numpy as np
def _seed_export_data(client):
project = client.post("/api/projects", json={"name": "Export Project"}).json()
@@ -58,7 +62,55 @@ def test_export_masks_zip(client):
assert response.status_code == 200
assert response.headers["content-type"].startswith("application/zip")
with zipfile.ZipFile(BytesIO(response.content)) as archive:
assert archive.namelist() == [f"mask_{annotation['id']:06d}.png"]
assert archive.namelist() == [
f"mask_{annotation['id']:06d}.png",
"semantic_frame_000000.png",
"semantic_classes.json",
]
def test_export_masks_uses_z_index_for_semantic_fusion(client):
project = client.post("/api/projects", json={"name": "Fusion Project"}).json()
frame = client.post(f"/api/projects/{project['id']}/frames", json={
"project_id": project["id"],
"frame_index": 0,
"image_url": "frames/0.jpg",
"width": 20,
"height": 20,
}).json()
low = client.post("/api/ai/annotate", json={
"project_id": project["id"],
"frame_id": frame["id"],
"mask_data": {
"polygons": [[[0.1, 0.1], [0.8, 0.1], [0.8, 0.8], [0.1, 0.8]]],
"label": "Low",
"color": "#00ff00",
"class": {"id": "low", "name": "Low", "color": "#00ff00", "zIndex": 10},
},
}).json()
high = client.post("/api/ai/annotate", json={
"project_id": project["id"],
"frame_id": frame["id"],
"mask_data": {
"polygons": [[[0.4, 0.4], [0.9, 0.4], [0.9, 0.9], [0.4, 0.9]]],
"label": "High",
"color": "#ff0000",
"class": {"id": "high", "name": "High", "color": "#ff0000", "zIndex": 20},
},
}).json()
response = client.get(f"/api/export/{project['id']}/masks")
assert response.status_code == 200
with zipfile.ZipFile(BytesIO(response.content)) as archive:
assert f"mask_{low['id']:06d}.png" in archive.namelist()
assert f"mask_{high['id']:06d}.png" in archive.namelist()
legend = json.loads(archive.read("semantic_classes.json"))
high_value = next(item["value"] for item in legend["classes"] if item["key"] == "class:high")
semantic_bytes = np.frombuffer(archive.read("semantic_frame_000000.png"), dtype=np.uint8)
semantic = cv2.imdecode(semantic_bytes, cv2.IMREAD_GRAYSCALE)
assert semantic[10, 10] == high_value
def test_export_missing_project_returns_404(client):

22
backend/tests/test_media.py
View File

@@ -140,3 +140,25 @@ def test_parse_task_runner_registers_frames(client, db_session, monkeypatch, tmp
assert project_detail["status"] == "ready"
frames = client.get(f"/api/projects/{project['id']}/frames").json()
assert "frame_000001.jpg" in frames[0]["image_url"]
def test_parse_task_runner_skips_already_cancelled_task(db_session):
from models import ProcessingTask
from services.media_task_runner import run_parse_media_task
task = ProcessingTask(
task_type="parse_video",
status="cancelled",
progress=100,
message="任务已取消",
project_id=1,
payload={"source_type": "video"},
)
db_session.add(task)
db_session.commit()
db_session.refresh(task)
result = run_parse_media_task(db_session, task.id)
assert result["status"] == "cancelled"
assert result["message"] == "任务已取消"

19
backend/tests/test_progress_events.py
View File

@@ -26,6 +26,25 @@ def test_task_progress_payload_uses_dashboard_task_id_and_project_name():
assert payload["status"] == "解析完成"
def test_task_progress_payload_marks_cancelled_tasks():
task = SimpleNamespace(
id=13,
project_id=7,
project=SimpleNamespace(name="demo.mp4"),
status="cancelled",
progress=100,
message="任务已取消",
error="Cancelled by user",
updated_at=None,
)
payload = task_progress_payload(task)
assert payload["type"] == "cancelled"
assert payload["status"] == "任务已取消"
assert payload["error"] == "Cancelled by user"
def test_publish_progress_event_writes_json_to_redis(monkeypatch):
calls = []

112
backend/tests/test_sam3_engine.py Normal file
View File

@@ -0,0 +1,112 @@
import json
from pathlib import Path
import numpy as np
from services.sam3_engine import SAM3Engine
class _Completed:
def __init__(self, returncode=0, stdout="", stderr=""):
self.returncode = returncode
self.stdout = stdout
self.stderr = stderr
def _external_settings(monkeypatch, python_path: Path):
python_path.write_text("#!/usr/bin/env python\n", encoding="utf-8")
python_path.chmod(0o755)
monkeypatch.setattr("services.sam3_engine.SAM3_PACKAGE_AVAILABLE", False)
monkeypatch.setattr("services.sam3_engine.TORCH_AVAILABLE", False)
monkeypatch.setattr("services.sam3_engine.settings.sam3_external_enabled", True)
monkeypatch.setattr("services.sam3_engine.settings.sam3_external_python", str(python_path))
monkeypatch.setattr("services.sam3_engine.settings.sam3_timeout_seconds", 10)
monkeypatch.setattr("services.sam3_engine.settings.sam3_status_cache_seconds", 30)
monkeypatch.setattr("services.sam3_engine.settings.sam3_confidence_threshold", 0.4)
def test_sam3_status_reports_external_runtime_ready(tmp_path, monkeypatch):
_external_settings(monkeypatch, tmp_path / "python")
def fake_run(args, **_kwargs):
assert "--status" in args
return _Completed(stdout=json.dumps({
"available": True,
"package_available": True,
"python_ok": True,
"torch_ok": True,
"cuda_available": True,
"device": "cuda",
"message": "ready",
}))
monkeypatch.setattr("services.sam3_engine.subprocess.run", fake_run)
status = SAM3Engine().status()
assert status["available"] is True
assert status["external_available"] is True
assert status["package_available"] is True
assert status["python_ok"] is True
assert status["message"] == "SAM 3 external runtime is ready; model will load in the helper process on inference."
def test_sam3_predict_semantic_uses_external_worker(tmp_path, monkeypatch):
_external_settings(monkeypatch, tmp_path / "python")
calls = []
def fake_run(args, **_kwargs):
calls.append(args)
if "--status" in args:
return _Completed(stdout=json.dumps({
"available": True,
"package_available": True,
"python_ok": True,
"torch_ok": True,
"cuda_available": True,
"device": "cuda",
"message": "ready",
}))
request_path = Path(args[-1])
request = json.loads(request_path.read_text(encoding="utf-8"))
assert request["text"] == "vessel"
assert request["confidence_threshold"] == 0.4
assert Path(request["image_path"]).exists()
return _Completed(stdout=json.dumps({
"polygons": [[[0.1, 0.1], [0.9, 0.1], [0.9, 0.9]]],
"scores": [0.91],
}))
monkeypatch.setattr("services.sam3_engine.subprocess.run", fake_run)
polygons, scores = SAM3Engine().predict_semantic(np.zeros((8, 8, 3), dtype=np.uint8), " vessel ")
assert polygons == [[[0.1, 0.1], [0.9, 0.1], [0.9, 0.9]]]
assert scores == [0.91]
assert any("--request" in args for args in calls)
def test_sam3_predict_semantic_reports_external_errors(tmp_path, monkeypatch):
_external_settings(monkeypatch, tmp_path / "python")
def fake_run(args, **_kwargs):
if "--status" in args:
return _Completed(stdout=json.dumps({
"available": True,
"package_available": True,
"python_ok": True,
"torch_ok": True,
"cuda_available": True,
"device": "cuda",
"message": "ready",
}))
return _Completed(returncode=1, stderr=json.dumps({"error": "HF access denied"}))
monkeypatch.setattr("services.sam3_engine.subprocess.run", fake_run)
try:
SAM3Engine().predict_semantic(np.zeros((8, 8, 3), dtype=np.uint8), "vessel")
except RuntimeError as exc:
assert "HF access denied" in str(exc)
else:
raise AssertionError("Expected SAM 3 external inference failure.")

104
backend/tests/test_tasks.py Normal file
View File

@@ -0,0 +1,104 @@
from models import ProcessingTask
def test_cancel_task_revokes_celery_and_updates_project(client, db_session, monkeypatch):
project = client.post("/api/projects", json={
"name": "Cancelable",
"video_path": "uploads/1/clip.mp4",
"status": "parsing",
}).json()
task = ProcessingTask(
task_type="parse_video",
status="running",
progress=35,
message="正在使用 FFmpeg/OpenCV 拆帧",
project_id=project["id"],
celery_task_id="celery-1",
payload={"source_type": "video"},
)
db_session.add(task)
db_session.commit()
db_session.refresh(task)
revoked = []
published = []
monkeypatch.setattr(
"routers.tasks.celery_app.control.revoke",
lambda celery_id, terminate, signal: revoked.append((celery_id, terminate, signal)),
)
monkeypatch.setattr("routers.tasks.publish_task_progress_event", lambda event_task: published.append(event_task.status))
response = client.post(f"/api/tasks/{task.id}/cancel")
assert response.status_code == 200
body = response.json()
assert body["status"] == "cancelled"
assert body["progress"] == 100
assert body["message"] == "任务已取消"
assert body["error"] == "Cancelled by user"
assert revoked == [("celery-1", True, "SIGTERM")]
assert published == ["cancelled"]
assert client.get(f"/api/projects/{project['id']}").json()["status"] == "pending"
def test_retry_task_creates_fresh_parse_task(client, db_session, monkeypatch):
project = client.post("/api/projects", json={
"name": "Retryable",
"video_path": "uploads/2/clip.mp4",
"source_type": "video",
"status": "error",
}).json()
task = ProcessingTask(
task_type="parse_video",
status="failed",
progress=100,
message="解析失败",
error="ffmpeg failed",
project_id=project["id"],
payload={"source_type": "video"},
)
db_session.add(task)
db_session.commit()
db_session.refresh(task)
class FakeAsyncResult:
id = "celery-retry"
queued = []
published = []
monkeypatch.setattr("routers.tasks.parse_project_media.delay", lambda task_id: queued.append(task_id) or FakeAsyncResult())
monkeypatch.setattr("routers.tasks.publish_task_progress_event", lambda event_task: published.append((event_task.id, event_task.status)))
response = client.post(f"/api/tasks/{task.id}/retry")
assert response.status_code == 202
body = response.json()
assert body["id"] != task.id
assert body["status"] == "queued"
assert body["progress"] == 0
assert body["celery_task_id"] == "celery-retry"
assert body["payload"]["retry_of"] == task.id
assert queued == [body["id"]]
assert published[0] == (body["id"], "queued")
assert published[-1] == (body["id"], "queued")
assert client.get(f"/api/projects/{project['id']}").json()["status"] == "parsing"
def test_task_actions_reject_invalid_states(client, db_session):
project = client.post("/api/projects", json={
"name": "Done",
"video_path": "uploads/3/clip.mp4",
}).json()
task = ProcessingTask(
task_type="parse_video",
status="success",
progress=100,
project_id=project["id"],
payload={"source_type": "video"},
)
db_session.add(task)
db_session.commit()
db_session.refresh(task)
assert client.post(f"/api/tasks/{task.id}/cancel").status_code == 409
assert client.post(f"/api/tasks/{task.id}/retry").status_code == 409