admin/Pre_Seg_Server
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: 完善 SAM2.1 模型选择与标注工作流

Browse Source 
- 后端 SAM2 引擎新增 sam2.1_hiera_tiny、sam2.1_hiera_small、sam2.1_hiera_base_plus、sam2.1_hiera_large 四个变体定义，并按变体维护 checkpoint/config、image predictor、video predictor、加载状态、错误信息和真实状态回报。

- 后端 SAM registry 仅暴露当前产品启用的 SAM2.1 变体，保留 sam2 作为 tiny 兼容别名，拒绝 sam3 产品入口，并把 point、box、interactive、auto、propagate 都分发到所选 SAM2.1 变体。

- 后端默认配置和下载脚本切换到 SAM2.1 checkpoint 命名，支持 legacy SAM2 checkpoint fallback，并在状态消息中标出 fallback 使用情况。

- 前端全局 AI 模型状态新增 SAM2.1 tiny/small/base+/large 类型和默认 tiny，API 请求默认携带 sam2.1_hiera_tiny，AI 页面提供模型变体选择和所选模型状态展示。

- AI 智能分割页移除当前产品不使用的 SAM3/文本提示入口，保留正向点、反向点、框选和参数开关；AI 页只展示本页生成的候选 mask，并支持遮罩清晰度调节、候选 mask 上继续加正/反点、清空本页候选、推送到工作区编辑。

- 工作区和 Canvas 补强 SAM2 交互式细化链路：框选后正/反点继续细化同一个候选 mask，反向点请求启用背景过滤，空结果会移除被否定候选；AI 推送到工作区后保留选中态和未保存 draft mask。

- 工作区标注保存闭环补强：未保存 mask 可归档保存，dirty saved mask 可更新，保存后用后端 saved annotation 替换已提交 draft，清空/删除已保存 mask 时同步后端删除。

- Dashboard 任务进度区改为展示 queued、running、success、failed、cancelled 最近任务，处理中统计只计算 queued/running，并保留近期完成记录。

- 时间轴在顶部时间进度条和底部缩略图导航轴之间新增已编辑帧标记带，基于当前项目帧内 masks 标出已有编辑/标注的帧，并支持点击标记跳转。

- 前端测试覆盖 SAM2.1 变体选择、模型状态徽标、AI 页候选隔离、遮罩透明度、候选上追加正/反点、推送工作区保留选择、Canvas 交互式细化、VideoWorkspace 传播/保存、Dashboard 进度和时间轴已编辑帧标记。

- 后端测试覆盖 SAM2.1 变体状态、sam2 alias 兼容、sam3 禁用、semantic 禁用、传播标注保存、Dashboard 最近任务状态和 SAM3 历史测试跳过说明。

- README、AGENTS 和 doc 文档同步当前真实进度，更新 SAM2.1 变体、SAM3 禁用、接口契约、设计冻结、需求冻结、前端元素审计、实施计划、FastAPI docs 说明和测试矩阵。
This commit is contained in:
admin
2026-05-01 23:39:53 +08:00
parent 8a9247075e
commit 29a1a87e52
38 changed files with 1087 additions and 631 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
backend/tests/test_ai.py
View File

@@ -87,28 +87,14 @@ def test_predict_applies_crop_and_background_filter_options(client, monkeypatch)
assert all(0.0 <= coord <= 1.0 for point in polygon for coord in point)
def test_predict_box_and_semantic_fallback(client, monkeypatch):
def test_predict_box_and_rejects_semantic_prompt(client, monkeypatch):
_, frame, _ = _create_project_and_frame(client)
calls = {}
monkeypatch.setattr("routers.ai._load_frame_image", lambda frame: np.zeros((10, 10, 3), dtype=np.uint8))
monkeypatch.setattr("routers.ai.sam_registry.predict_box", lambda model, image, box: (
[[[0.2, 0.2], [0.8, 0.2], [0.8, 0.8]]],
[0.8],
))
def fake_predict_semantic(model, image, text, confidence_threshold=None):
calls["semantic"] = {
"model": model,
"text": text,
"confidence_threshold": confidence_threshold,
}
return (
[[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0]]],
[0.5],
)
monkeypatch.setattr("routers.ai.sam_registry.predict_semantic", fake_predict_semantic)
box_response = client.post("/api/ai/predict", json={
"image_id": frame["id"],
"prompt_type": "box",
@@ -124,13 +110,8 @@ def test_predict_box_and_semantic_fallback(client, monkeypatch):
assert box_response.status_code == 200
assert box_response.json()["scores"] == [0.8]
assert semantic_response.status_code == 200
assert semantic_response.json()["scores"] == [0.5]
assert calls["semantic"] == {
"model": "sam3",
"text": "胆囊",
"confidence_threshold": 0.05,
}
assert semantic_response.status_code == 400
assert "Unsupported model: sam3" in semantic_response.json()["detail"]
def test_predict_interactive_combines_box_and_points(client, monkeypatch):
@@ -158,13 +139,13 @@ def test_predict_interactive_combines_box_and_points(client, monkeypatch):
"points": [[0.5, 0.5], [0.2, 0.2]],
"labels": [1, 0],
},
"model": "sam2",
"model": "sam2.1_hiera_small",
})
assert response.status_code == 200
assert response.json()["scores"] == [0.88]
assert calls == {
"model": "sam2",
"model": "sam2.1_hiera_small",
"box": [0.1, 0.1, 0.9, 0.9],
"points": [[0.5, 0.5], [0.2, 0.2]],
"labels": [1, 0],
@@ -173,7 +154,7 @@ def test_predict_interactive_combines_box_and_points(client, monkeypatch):
def test_model_status_reports_runtime(client, monkeypatch):
monkeypatch.setattr("routers.ai.sam_registry.runtime_status", lambda selected_model=None: {
"selected_model": selected_model or "sam2",
"selected_model": "sam2.1_hiera_tiny",
"gpu": {
"available": False,
"device": "cpu",
@@ -184,8 +165,8 @@ def test_model_status_reports_runtime(client, monkeypatch):
},
"models": [
{
"id": "sam2",
"label": "SAM 2",
"id": "sam2.1_hiera_tiny",
"label": "SAM 2.1 Tiny",
"available": True,
"loaded": False,
"device": "cpu",
@@ -198,31 +179,23 @@ def test_model_status_reports_runtime(client, monkeypatch):
"torch_ok": True,
"cuda_required": False,
},
{
"id": "sam3",
"label": "SAM 3",
"available": False,
"loaded": False,
"device": "unavailable",
"supports": ["semantic"],
"message": "missing Python 3.12+ runtime",
"package_available": False,
"checkpoint_exists": False,
"checkpoint_path": None,
"python_ok": False,
"torch_ok": True,
"cuda_required": True,
},
],
})
response = client.get("/api/ai/models/status?selected_model=sam3")
response = client.get("/api/ai/models/status")
assert response.status_code == 200
body = response.json()
assert body["selected_model"] == "sam3"
assert body["models"][1]["id"] == "sam3"
assert body["models"][1]["available"] is False
assert body["selected_model"] == "sam2.1_hiera_tiny"
assert len(body["models"]) == 1
assert body["models"][0]["id"] == "sam2.1_hiera_tiny"
def test_model_status_rejects_disabled_sam3(client):
response = client.get("/api/ai/models/status?selected_model=sam3")
assert response.status_code == 400
assert "Unsupported model" in response.json()["detail"]
def test_propagate_saves_tracked_annotations(client, monkeypatch):
@@ -267,7 +240,7 @@ def test_propagate_saves_tracked_annotations(client, monkeypatch):
response = client.post("/api/ai/propagate", json={
"project_id": project["id"],
"frame_id": frames[0]["id"],
"model": "sam2",
"model": "sam2.1_hiera_tiny",
"direction": "forward",
"max_frames": 2,
"include_source": False,
@@ -285,13 +258,13 @@ def test_propagate_saves_tracked_annotations(client, monkeypatch):
body = response.json()
assert body["created_annotation_count"] == 1
assert body["processed_frame_count"] == 2
assert calls["model"] == "sam2"
assert calls["model"] == "sam2.1_hiera_tiny"
assert calls["source_frame_index"] == 0
assert calls["direction"] == "forward"
assert calls["frame_count"] == 2
saved = body["annotations"][0]
assert saved["frame_id"] == frames[1]["id"]
assert saved["mask_data"]["source"] == "sam2_propagation"
assert saved["mask_data"]["source"] == "sam2.1_hiera_tiny_propagation"
assert saved["mask_data"]["class"]["name"] == "胆囊"
assert saved["mask_data"]["score"] == 0.8

41
backend/tests/test_dashboard.py
View File

@@ -69,3 +69,44 @@ def test_dashboard_overview_uses_persisted_records(client, db_session):
assert any(item["kind"] == "annotation" for item in body["activity"])
assert any(item["kind"] == "template" for item in body["activity"])
assert all(item["name"] != "Ready Project" for item in body["tasks"])
def test_dashboard_overview_keeps_recent_success_tasks_in_progress_list(client, db_session):
from models import ProcessingTask
project = client.post("/api/projects", json={
"name": "Completed Project",
"status": "ready",
}).json()
task = ProcessingTask(
task_type="parse_video",
status="success",
progress=100,
message="解析完成",
project_id=project["id"],
payload={"source_type": "video"},
result={"frames_extracted": 120},
)
db_session.add(task)
db_session.commit()
db_session.refresh(task)
response = client.get("/api/dashboard/overview")
assert response.status_code == 200
body = response.json()
assert body["summary"]["parsing_task_count"] == 0
assert body["tasks"] == [
{
"id": f"task-{task.id}",
"task_id": task.id,
"project_id": project["id"],
"name": "Completed Project",
"progress": 100,
"status": "解析完成",
"raw_status": "success",
"frame_count": 120,
"error": None,
"updated_at": body["tasks"][0]["updated_at"],
},
]

25
backend/tests/test_sam2_engine.py
View File

@@ -1,6 +1,6 @@
import numpy as np
from services.sam2_engine import SAM2Engine
from services.sam2_engine import DEFAULT_SAM2_MODEL_ID, SAM2Engine
class _FakePredictor:
@@ -26,8 +26,8 @@ def _mask(offset=0):
def _ready_engine(monkeypatch, predictor):
monkeypatch.setattr("services.sam2_engine.SAM2_AVAILABLE", True)
engine = SAM2Engine()
engine._model_loaded = True
engine._predictor = predictor
engine._model_loaded[DEFAULT_SAM2_MODEL_ID] = True
engine._predictors[DEFAULT_SAM2_MODEL_ID] = predictor
return engine
@@ -39,6 +39,7 @@ def test_sam2_point_prediction_requests_single_best_mask(monkeypatch):
engine = _ready_engine(monkeypatch, predictor)
polygons, scores = engine.predict_points(
DEFAULT_SAM2_MODEL_ID,
np.zeros((32, 32, 3), dtype=np.uint8),
[[0.5, 0.5]],
[1],
@@ -56,8 +57,24 @@ def test_sam2_auto_prediction_keeps_single_best_mask(monkeypatch):
)
engine = _ready_engine(monkeypatch, predictor)
polygons, scores = engine.predict_auto(np.zeros((32, 32, 3), dtype=np.uint8))
polygons, scores = engine.predict_auto(DEFAULT_SAM2_MODEL_ID, np.zeros((32, 32, 3), dtype=np.uint8))
assert predictor.calls[0]["multimask_output"] is False
assert len(polygons) == 1
assert scores == [0.800000011920929]
def test_sam2_status_exposes_selectable_variants(monkeypatch, tmp_path):
checkpoint = tmp_path / "sam2.1_hiera_small.pt"
checkpoint.write_bytes(b"model")
monkeypatch.setattr("services.sam2_engine.settings.sam_model_path", str(tmp_path / "sam2.1_hiera_tiny.pt"))
engine = SAM2Engine()
status = engine.status("sam2.1_hiera_small")
assert engine.normalize_model_id("sam2") == DEFAULT_SAM2_MODEL_ID
assert "sam2.1_hiera_small" in engine.variant_ids()
assert status["id"] == "sam2.1_hiera_small"
assert status["label"] == "SAM 2.1 Small"
assert status["checkpoint_exists"] is True
assert status["checkpoint_path"].endswith("sam2.1_hiera_small.pt")

6
backend/tests/test_sam3_engine.py
View File

@@ -2,6 +2,12 @@ import json
from pathlib import Path
import numpy as np
import pytest
pytest.skip(
"SAM 3 integration is disabled in the current SAM2-only product flow.",
allow_module_level=True,
)
from services.sam3_engine import SAM3Engine
from services.sam3_external_worker import _prediction_to_response, _to_numpy