feat: 建立 SAM2 标注闭环基线
- 打通工作区真实标注闭环:支持手工多边形、矩形、圆形、点区域和线段生成 mask,并可保存、回显、更新和删除后端 annotation。 - 增强 polygon 编辑器:支持顶点拖动、顶点删除、边中点插入、多 polygon 子区域选择编辑,以及区域合并和区域去除。 - 接入 GT mask 导入:后端支持二值/多类别 mask 拆分、contour 转 polygon、distance transform seed point,前端支持导入、回显和 seed point 拖动编辑。 - 完善导出能力:COCO JSON 导出对齐前端,PNG mask ZIP 同时包含单标注 mask、按 zIndex 融合的 semantic_frame 和 semantic_classes.json。 - 打通异步任务管理:新增任务取消、重试、失败详情接口与 Dashboard 控件,worker 支持取消状态检查并通过 Redis/WebSocket 推送 cancelled 事件。 - 对接 Dashboard 后端数据:概览统计、解析队列和实时流转记录从 FastAPI 聚合接口与 WebSocket 更新。 - 增强 AI 推理参数:前端发送 crop_to_prompt、auto_filter_background 和 min_score,后端支持点/框 prompt 局部裁剪推理、结果回映射和负向点/低分过滤。 - 接入 SAM3 基础设施:新增独立 Python 3.12 sam3 环境安装脚本、外部 worker helper、后端桥接和真实 Python/CUDA/包/HF checkpoint access 状态检测。 - 保留 SAM3 授权边界:当前官方 facebook/sam3 gated 权重未授权时状态接口会返回不可用,不伪装成可推理。 - 增强前端状态管理:新增 mask undo/redo 历史栈、AI 模型选择状态、保存状态 dirty/draft/saved 流转和项目状态归一化。 - 更新前端 API 封装:补充 annotation CRUD、GT mask import、mask ZIP export、task cancel/retry/detail、AI runtime status 和 prediction options。 - 更新 UI 控件:ToolsPalette、AISegmentation、VideoWorkspace 和 CanvasArea 接入真实操作、导入导出、撤销重做、任务控制和模型状态。 - 新增 polygon-clipping 依赖,用于前端区域 union/difference 几何运算。 - 完善后端 schemas/status/progress:补充 AI 模型外部状态字段、任务 cancelled 状态和进度事件 payload。 - 补充测试覆盖:新增后端任务控制、SAM3 桥接、GT mask、导出融合、AI options 测试;补充前端 Canvas、Dashboard、VideoWorkspace、ToolsPalette、API 和 store 测试。 - 更新 README、AGENTS 和 doc 文档:冻结当前需求/设计/测试计划,标注真实功能、剩余 Mock、SAM3 授权边界和后续实施顺序。
This commit is contained in:
@@ -9,8 +9,14 @@ the package.
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
@@ -41,6 +47,8 @@ class SAM3Engine:
|
||||
self._processor: Any | None = None
|
||||
self._model_loaded = False
|
||||
self._last_error: str | None = None
|
||||
self._external_status_cache: dict[str, Any] | None = None
|
||||
self._external_status_checked_at = 0.0
|
||||
|
||||
def _python_ok(self) -> bool:
|
||||
return sys.version_info >= (3, 12)
|
||||
@@ -51,6 +59,81 @@ class SAM3Engine:
|
||||
def _can_load(self) -> bool:
|
||||
return bool(SAM3_PACKAGE_AVAILABLE and TORCH_AVAILABLE and self._python_ok() and self._gpu_ok())
|
||||
|
||||
def _worker_path(self) -> Path:
|
||||
return Path(__file__).with_name("sam3_external_worker.py")
|
||||
|
||||
def _external_python_exists(self) -> bool:
|
||||
return bool(settings.sam3_external_enabled and os.path.isfile(settings.sam3_external_python))
|
||||
|
||||
def _external_status(self, force: bool = False) -> dict[str, Any]:
|
||||
now = time.monotonic()
|
||||
if (
|
||||
not force
|
||||
and self._external_status_cache is not None
|
||||
and now - self._external_status_checked_at < settings.sam3_status_cache_seconds
|
||||
):
|
||||
return self._external_status_cache
|
||||
|
||||
if not settings.sam3_external_enabled:
|
||||
status = {
|
||||
"available": False,
|
||||
"package_available": False,
|
||||
"python_ok": False,
|
||||
"torch_ok": False,
|
||||
"cuda_available": False,
|
||||
"device": "unavailable",
|
||||
"message": "SAM 3 external runtime is disabled.",
|
||||
}
|
||||
elif not self._external_python_exists():
|
||||
status = {
|
||||
"available": False,
|
||||
"package_available": False,
|
||||
"python_ok": False,
|
||||
"torch_ok": False,
|
||||
"cuda_available": False,
|
||||
"device": "unavailable",
|
||||
"message": f"SAM 3 external Python not found: {settings.sam3_external_python}",
|
||||
}
|
||||
else:
|
||||
try:
|
||||
env = os.environ.copy()
|
||||
env["SAM3_MODEL_VERSION"] = settings.sam3_model_version
|
||||
completed = subprocess.run(
|
||||
[settings.sam3_external_python, str(self._worker_path()), "--status"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=min(settings.sam3_timeout_seconds, 30),
|
||||
check=False,
|
||||
env=env,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
detail = completed.stderr.strip() or completed.stdout.strip()
|
||||
status = {
|
||||
"available": False,
|
||||
"package_available": False,
|
||||
"python_ok": False,
|
||||
"torch_ok": False,
|
||||
"cuda_available": False,
|
||||
"device": "unavailable",
|
||||
"message": f"SAM 3 external status failed: {detail}",
|
||||
}
|
||||
else:
|
||||
status = json.loads(completed.stdout)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
status = {
|
||||
"available": False,
|
||||
"package_available": False,
|
||||
"python_ok": False,
|
||||
"torch_ok": False,
|
||||
"cuda_available": False,
|
||||
"device": "unavailable",
|
||||
"message": f"SAM 3 external status failed: {exc}",
|
||||
}
|
||||
|
||||
self._external_status_cache = status
|
||||
self._external_status_checked_at = now
|
||||
return status
|
||||
|
||||
def _load_model(self) -> None:
|
||||
if self._model_loaded:
|
||||
return
|
||||
@@ -92,26 +175,86 @@ class SAM3Engine:
|
||||
return "SAM 3 dependencies are present; model will load on first inference."
|
||||
|
||||
def status(self) -> dict:
|
||||
available = self._can_load()
|
||||
external_status = self._external_status()
|
||||
available = bool(self._can_load() or external_status.get("available"))
|
||||
external_ready = bool(external_status.get("available"))
|
||||
message = self._last_error or self._status_message()
|
||||
if self._processor is not None:
|
||||
message = "SAM 3 model loaded and ready."
|
||||
elif external_ready:
|
||||
message = "SAM 3 external runtime is ready; model will load in the helper process on inference."
|
||||
elif external_status.get("message") and not self._can_load():
|
||||
message = str(external_status["message"])
|
||||
return {
|
||||
"id": "sam3",
|
||||
"label": "SAM 3",
|
||||
"available": available,
|
||||
"loaded": self._processor is not None,
|
||||
"device": "cuda" if self._gpu_ok() else "unavailable",
|
||||
"device": "cuda" if self._gpu_ok() else str(external_status.get("device", "unavailable")),
|
||||
"supports": ["semantic"],
|
||||
"message": "SAM 3 model loaded and ready." if self._processor is not None else (self._last_error or self._status_message()),
|
||||
"package_available": SAM3_PACKAGE_AVAILABLE,
|
||||
"checkpoint_exists": SAM3_PACKAGE_AVAILABLE,
|
||||
"message": message,
|
||||
"package_available": bool(SAM3_PACKAGE_AVAILABLE or external_status.get("package_available")),
|
||||
"checkpoint_exists": bool(SAM3_PACKAGE_AVAILABLE or external_status.get("checkpoint_access")),
|
||||
"checkpoint_path": f"official/HuggingFace ({settings.sam3_model_version})",
|
||||
"python_ok": self._python_ok(),
|
||||
"torch_ok": TORCH_AVAILABLE,
|
||||
"python_ok": bool(self._python_ok() or external_status.get("python_ok")),
|
||||
"torch_ok": bool(TORCH_AVAILABLE or external_status.get("torch_ok")),
|
||||
"cuda_required": True,
|
||||
"external_available": external_ready,
|
||||
"external_python": settings.sam3_external_python if settings.sam3_external_enabled else None,
|
||||
}
|
||||
|
||||
def _predict_semantic_external(self, image: np.ndarray, text: str) -> tuple[list[list[list[float]]], list[float]]:
|
||||
status = self._external_status(force=True)
|
||||
if not status.get("available"):
|
||||
raise RuntimeError(status.get("message") or "SAM 3 external runtime is unavailable.")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="sam3_") as tmpdir:
|
||||
tmp_path = Path(tmpdir)
|
||||
image_path = tmp_path / "image.png"
|
||||
request_path = tmp_path / "request.json"
|
||||
Image.fromarray(image).save(image_path)
|
||||
request_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"image_path": str(image_path),
|
||||
"text": text.strip(),
|
||||
"model_version": settings.sam3_model_version,
|
||||
"confidence_threshold": settings.sam3_confidence_threshold,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
env = os.environ.copy()
|
||||
env["SAM3_MODEL_VERSION"] = settings.sam3_model_version
|
||||
completed = subprocess.run(
|
||||
[settings.sam3_external_python, str(self._worker_path()), "--request", str(request_path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=settings.sam3_timeout_seconds,
|
||||
check=False,
|
||||
env=env,
|
||||
)
|
||||
|
||||
if completed.returncode != 0:
|
||||
detail = completed.stderr.strip() or completed.stdout.strip()
|
||||
try:
|
||||
parsed = json.loads(detail)
|
||||
detail = parsed.get("error", detail)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
raise RuntimeError(f"SAM 3 external inference failed: {detail}")
|
||||
|
||||
payload = json.loads(completed.stdout)
|
||||
if payload.get("error"):
|
||||
raise RuntimeError(str(payload["error"]))
|
||||
return payload.get("polygons", []), payload.get("scores", [])
|
||||
|
||||
def predict_semantic(self, image: np.ndarray, text: str) -> tuple[list[list[list[float]]], list[float]]:
|
||||
if not text.strip():
|
||||
raise ValueError("SAM 3 semantic prompt requires non-empty text.")
|
||||
if not self._can_load() and self._external_status().get("available"):
|
||||
return self._predict_semantic_external(image, text)
|
||||
if not self._ensure_ready():
|
||||
raise RuntimeError(self.status()["message"])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user