feat: 完善 AI 分割与工作区标注闭环
功能增加: - 将视频导入和生成帧拆成两个明确动作,项目库生成帧时选择 FPS,工作区不再自动触发拆帧。 - 为工作区新增调整多边形工具,支持选中 mask、拖动顶点、边中点插点、双击边界按位置插点,并保留多 polygon 子区域编辑。 - 打通 AI 页 SAM2/SAM3 结果到工作区的联动,生成 mask 后自动选中,可在右侧分类树换标签,并推送到工作区继续编辑。 - 增强 Dashboard WebSocket 连接状态与心跳,使用真实 onopen/onclose/onerror 状态驱动前端显示。 - 完善 SAM3 external worker 适配,支持 box prompt、semantic 请求级阈值和 video tracker 路径。 bugfix: - 修复 SAM2 文本语义误走自动分割的问题,改为提示使用点提示或切换 SAM3。 - 修复 SAM2 多候选重叠显示的问题,点提示和 auto fallback 默认只采用最高分候选。 - 修复 SAM2 反向点看起来无效的问题,带负点时启用背景过滤,过滤为空时移除旧候选。 - 修复 SAM3 单个 2D mask 结果无法转 polygon、低阈值 semantic 返回被默认阈值吞掉的问题。 - 修复 AI 页 mask 未选中导致分类树无法修改 SAM2 结果标签的问题。 测试和文档: - 补充 CanvasArea、AISegmentation、ProjectLibrary、VideoWorkspace、Dashboard、websocket 和 SAM engine/API 测试。 - 新增 backend/tests/test_sam2_engine.py,覆盖 SAM2 单候选请求和 auto fallback 行为。 - 更新 README、AGENTS 和 doc 需求/设计/接口/测试矩阵,按当前实现冻结功能状态。
This commit is contained in:
@@ -207,7 +207,7 @@ class SAM2Engine:
|
||||
masks, scores, _ = self._predictor.predict(
|
||||
point_coords=pts,
|
||||
point_labels=lbls,
|
||||
multimask_output=True,
|
||||
multimask_output=False,
|
||||
)
|
||||
|
||||
polygons = []
|
||||
@@ -335,16 +335,16 @@ class SAM2Engine:
|
||||
masks, scores, _ = self._predictor.predict(
|
||||
point_coords=pts,
|
||||
point_labels=lbls,
|
||||
multimask_output=True,
|
||||
multimask_output=False,
|
||||
)
|
||||
|
||||
polygons = []
|
||||
for m in masks[:3]: # Limit to top 3 masks
|
||||
for m in masks[:1]:
|
||||
poly = self._mask_to_polygon(m)
|
||||
if poly:
|
||||
polygons.append(poly)
|
||||
|
||||
return polygons, scores[:3].tolist()
|
||||
return polygons, scores[:1].tolist()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("SAM2 auto prediction failed: %s", exc)
|
||||
return self._dummy_polygons(image.shape[1], image.shape[0]), [0.5]
|
||||
|
||||
@@ -260,6 +260,7 @@ class SAM3Engine:
|
||||
*,
|
||||
text: str = "",
|
||||
box: list[float] | None = None,
|
||||
confidence_threshold: float | None = None,
|
||||
) -> tuple[list[list[list[float]]], list[float]]:
|
||||
status = self._external_status(force=True)
|
||||
if not status.get("available"):
|
||||
@@ -279,7 +280,11 @@ class SAM3Engine:
|
||||
"box": box,
|
||||
"model_version": settings.sam3_model_version,
|
||||
"checkpoint_path": self._checkpoint_path(),
|
||||
"confidence_threshold": settings.sam3_confidence_threshold,
|
||||
"confidence_threshold": (
|
||||
confidence_threshold
|
||||
if confidence_threshold is not None
|
||||
else settings.sam3_confidence_threshold
|
||||
),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
@@ -312,8 +317,18 @@ class SAM3Engine:
|
||||
raise RuntimeError(str(payload["error"]))
|
||||
return payload.get("polygons", []), payload.get("scores", [])
|
||||
|
||||
def _predict_semantic_external(self, image: np.ndarray, text: str) -> tuple[list[list[list[float]]], list[float]]:
|
||||
return self._predict_external(image, "semantic", text=text)
|
||||
def _predict_semantic_external(
|
||||
self,
|
||||
image: np.ndarray,
|
||||
text: str,
|
||||
confidence_threshold: float | None = None,
|
||||
) -> tuple[list[list[list[float]]], list[float]]:
|
||||
return self._predict_external(
|
||||
image,
|
||||
"semantic",
|
||||
text=text,
|
||||
confidence_threshold=confidence_threshold,
|
||||
)
|
||||
|
||||
def _predict_box_external(self, image: np.ndarray, box: list[float]) -> tuple[list[list[list[float]]], list[float]]:
|
||||
return self._predict_external(image, "box", box=box)
|
||||
@@ -378,11 +393,16 @@ class SAM3Engine:
|
||||
raise RuntimeError(str(payload["error"]))
|
||||
return payload.get("frames", [])
|
||||
|
||||
def predict_semantic(self, image: np.ndarray, text: str) -> tuple[list[list[list[float]]], list[float]]:
|
||||
def predict_semantic(
|
||||
self,
|
||||
image: np.ndarray,
|
||||
text: str,
|
||||
confidence_threshold: float | None = None,
|
||||
) -> tuple[list[list[list[float]]], list[float]]:
|
||||
if not text.strip():
|
||||
raise ValueError("SAM 3 semantic prompt requires non-empty text.")
|
||||
if not self._can_load() and self._external_status().get("available"):
|
||||
return self._predict_semantic_external(image, text)
|
||||
return self._predict_semantic_external(image, text, confidence_threshold=confidence_threshold)
|
||||
if not self._ensure_ready():
|
||||
raise RuntimeError(self.status()["message"])
|
||||
|
||||
|
||||
@@ -190,7 +190,9 @@ def _video_outputs_to_response(outputs: dict[str, Any]) -> dict[str, Any]:
|
||||
def _prediction_to_response(output: dict[str, Any]) -> dict[str, Any]:
|
||||
masks = _to_numpy(output.get("masks", []))
|
||||
scores = _to_numpy(output.get("scores", []))
|
||||
if masks.ndim == 4:
|
||||
if masks.ndim == 2:
|
||||
masks = masks[None, :, :]
|
||||
elif masks.ndim == 4:
|
||||
masks = masks[:, 0]
|
||||
elif masks.ndim == 3 and masks.shape[0] == 1:
|
||||
masks = masks[None, 0]
|
||||
|
||||
@@ -83,10 +83,20 @@ class SAMRegistry:
|
||||
def predict_auto(self, model_id: str | None, image: Any):
|
||||
return self._ensure_available(model_id).predict_auto(image)
|
||||
|
||||
def predict_semantic(self, model_id: str | None, image: Any, text: str):
|
||||
def predict_semantic(
|
||||
self,
|
||||
model_id: str | None,
|
||||
image: Any,
|
||||
text: str,
|
||||
confidence_threshold: float | None = None,
|
||||
):
|
||||
model = self.normalize_model_id(model_id)
|
||||
if model == "sam3":
|
||||
return self._ensure_available(model).predict_semantic(image, text)
|
||||
return self._ensure_available(model).predict_semantic(
|
||||
image,
|
||||
text,
|
||||
confidence_threshold=confidence_threshold,
|
||||
)
|
||||
return self._ensure_available(model).predict_auto(image)
|
||||
|
||||
def propagate_video(
|
||||
|
||||
Reference in New Issue
Block a user