admin/Pre_Seg_Server
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: 完善 mask 编辑、传播平滑与开发重启闭环

Browse Source 
功能增加：

- 新增后端 /api/ai/smooth-mask 接口，对当前 mask polygon 执行 Chaikin 边缘平滑，并返回 polygon、bbox、area 与拓扑锚点。

- 在右侧实例属性面板加入边缘平滑强度和应用边缘平滑操作，应用后将 mask 标记为 draft/dirty，并通过正常保存链路落库。

- 保存标注与传播 seed 时保留 geometry_smoothing 元数据，自动传播 forward/backward 结果保存前应用同一平滑参数。

- 自动传播 seed signature 纳入平滑参数，修改平滑强度后会触发旧同源传播结果清理并重新传播。

- 支持跨帧跟随同一传播链 mask，AI 推送回工作区时保留当前帧视角。

Bugfix：

- 修复中间帧向前传播时旧 forward/backward 同物体结果未被清理导致双重 mask 的问题。

- 修复 propagation worker 写入目标帧前只按旧方向清理导致 backward 重传残留的问题。

- 修复多边形顶点拖拽和编辑后画布视口异常移动的问题，并补充拖拽状态回写。

- 修复实例属性标题跟随全局 active class 而不是当前 mask label 的问题，并移除后端模型置信度展示。

开发与部署：

- 新增 restart_dev_services.sh，使用 setsid 独立后台重启 FastAPI、Celery 和前端，写入 pid/log 文件并做 3000/8000 健康检查。

- 明确后端或 Celery 相关改动完成后需要运行重启脚本，保证运行态加载最新代码。

测试与文档：

- 补充后端 smooth-mask、传播平滑 metadata、seed signature、传播去重方向覆盖等测试。

- 补充前端 OntologyInspector、VideoWorkspace、CanvasArea 和 api 契约测试，覆盖边缘平滑、传播参数、跨帧选区跟随和画布编辑行为。

- 更新 README、AGENTS、安装文档、前端元素审计、需求冻结、设计冻结和测试计划，记录当前真实行为与重启要求。
This commit is contained in:
admin
2026-05-02 17:04:02 +08:00
parent f365539ff2
commit 4c1d3dba73
20 changed files with 1358 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
src/components/CanvasArea.test.tsx
View File

@@ -381,6 +381,91 @@ describe('CanvasArea', () => {
.filter((element) => element.getAttribute('data-fill') === '#ffffff')).toHaveLength(3);
});
it('selects the linked propagated mask when switching from the seed frame', async () => {
const propagatedFrame = { ...frame, id: 'frame-2', index: 1, url: '/frame-2.jpg' };
useStore.setState({
selectedMaskIds: ['annotation-7'],
masks: [
{
id: 'annotation-7',
annotationId: '7',
frameId: 'frame-1',
pathData: 'M 0 0 L 10 0 L 10 10 Z',
label: '胆囊',
color: '#facc15',
segmentation: [[0, 0, 10, 0, 10, 10]],
},
{
id: 'annotation-20',
annotationId: '20',
frameId: 'frame-2',
pathData: 'M 1 1 L 11 1 L 11 11 Z',
label: '胆囊',
color: '#facc15',
segmentation: [[1, 1, 11, 1, 11, 11]],
metadata: {
source: 'sam2.1_hiera_tiny_propagation',
source_annotation_id: 7,
source_mask_id: 'annotation-7',
propagation_seed_key: 'annotation:7',
},
},
],
});
const { rerender } = render(<CanvasArea activeTool="edit_polygon" frame={frame} />);
rerender(<CanvasArea activeTool="edit_polygon" frame={propagatedFrame} />);
await waitFor(() => expect(useStore.getState().selectedMaskIds).toEqual(['annotation-20']));
expect(screen.getByText('当前图层: 胆囊 #20')).toBeInTheDocument();
});
it('keeps following the same propagation chain between propagated frames', async () => {
const propagatedFrame = { ...frame, id: 'frame-2', index: 1, url: '/frame-2.jpg' };
const laterPropagatedFrame = { ...frame, id: 'frame-3', index: 2, url: '/frame-3.jpg' };
useStore.setState({
selectedMaskIds: ['annotation-20'],
masks: [
{
id: 'annotation-20',
annotationId: '20',
frameId: 'frame-2',
pathData: 'M 1 1 L 11 1 L 11 11 Z',
label: '胆囊',
color: '#facc15',
segmentation: [[1, 1, 11, 1, 11, 11]],
metadata: {
source: 'sam2.1_hiera_tiny_propagation',
source_annotation_id: 7,
source_mask_id: 'annotation-7',
propagation_seed_key: 'annotation:7',
},
},
{
id: 'annotation-21',
annotationId: '21',
frameId: 'frame-3',
pathData: 'M 2 2 L 12 2 L 12 12 Z',
label: '胆囊',
color: '#facc15',
segmentation: [[2, 2, 12, 2, 12, 12]],
metadata: {
source: 'sam2.1_hiera_tiny_propagation',
source_annotation_id: 7,
source_mask_id: 'annotation-7',
propagation_seed_key: 'annotation:7',
},
},
],
});
const { rerender } = render(<CanvasArea activeTool="edit_polygon" frame={propagatedFrame} />);
rerender(<CanvasArea activeTool="edit_polygon" frame={laterPropagatedFrame} />);
await waitFor(() => expect(useStore.getState().selectedMaskIds).toEqual(['annotation-21']));
expect(screen.getByText('当前图层: 胆囊 #21')).toBeInTheDocument();
});
it('renders imported GT seed points for editable point regions', () => {
useStore.setState({
masks: [

107
src/components/CanvasArea.tsx
View File

@@ -31,6 +31,72 @@ function clamp(value: number, min: number, max: number): number {
return Math.min(Math.max(value, min), max);
}
function metadataNumber(value: unknown): number | null {
const parsed = Number(value);
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
}
function propagationSourceMaskTokens(value: unknown): string[] {
if (typeof value !== 'string' || value.length === 0) return [];
const tokens = [`mask:${value}`];
const annotationMatch = value.match(/^annotation-(\d+)$/);
if (annotationMatch) {
tokens.push(`annotation:${annotationMatch[1]}`);
}
return tokens;
}
function isPropagationMask(mask: Mask): boolean {
const metadata = mask.metadata || {};
const source = typeof metadata.source === 'string' ? metadata.source : '';
return source.includes('_propagation')
|| metadata.propagated_from_frame_id !== undefined
|| metadata.propagation_seed_key !== undefined
|| metadata.source_annotation_id !== undefined
|| metadata.source_mask_id !== undefined;
}
function propagationLineageTokens(mask: Mask): Set<string> {
const metadata = mask.metadata || {};
const tokens = new Set<string>([`mask:${mask.id}`]);
if (mask.annotationId) {
tokens.add(`annotation:${mask.annotationId}`);
}
const sourceAnnotationId = metadataNumber(metadata.source_annotation_id);
if (sourceAnnotationId !== null) {
tokens.add(`annotation:${sourceAnnotationId}`);
}
propagationSourceMaskTokens(metadata.source_mask_id).forEach((token) => tokens.add(token));
if (typeof metadata.propagation_seed_key === 'string' && metadata.propagation_seed_key.length > 0) {
tokens.add(`seed-key:${metadata.propagation_seed_key}`);
}
return tokens;
}
function findLinkedMasksOnFrame(selectedIds: string[], allMasks: Mask[], targetFrameId?: string): string[] {
if (!targetFrameId || selectedIds.length === 0) return [];
const selectedMasks = selectedIds
.map((id) => allMasks.find((mask) => mask.id === id))
.filter((mask): mask is Mask => Boolean(mask));
if (selectedMasks.length === 0) return [];
const selectedTokens = new Set<string>();
const selectedHasPropagation = selectedMasks.some(isPropagationMask);
selectedMasks.forEach((mask) => {
propagationLineageTokens(mask).forEach((token) => selectedTokens.add(token));
});
return allMasks
.filter((mask) => String(mask.frameId) === String(targetFrameId))
.filter((mask) => {
const candidateHasPropagation = isPropagationMask(mask);
if (!selectedHasPropagation && !candidateHasPropagation) return false;
const candidateTokens = propagationLineageTokens(mask);
return [...candidateTokens].some((token) => selectedTokens.has(token));
})
.map((mask) => mask.id);
}
function polygonPath(points: CanvasPoint[]): string {
if (points.length === 0) return '';
return points
@@ -425,11 +491,19 @@ export function CanvasArea({ activeTool, frame, onClearMasks, onDeleteMaskAnnota
useEffect(() => {
if (previousFrameIdRef.current === frame?.id) return;
previousFrameIdRef.current = frame?.id;
setSelectedMaskId(null);
setSelectedMaskIds([]);
const linkedMaskIds = findLinkedMasksOnFrame(useStore.getState().selectedMaskIds, masks, frame?.id);
if (linkedMaskIds.length > 0) {
setSelectedMaskId(linkedMaskIds[0]);
setSelectedMaskIds(linkedMaskIds);
setGlobalSelectedMaskIds(linkedMaskIds);
} else {
setSelectedMaskId(null);
setSelectedMaskIds([]);
setGlobalSelectedMaskIds([]);
}
setSelectedPolygonIndex(0);
setSelectedVertexIndex(null);
}, [frame?.id]);
}, [frame?.id, masks, setGlobalSelectedMaskIds]);
useEffect(() => {
setPoints([]);
@@ -439,16 +513,23 @@ export function CanvasArea({ activeTool, frame, onClearMasks, onDeleteMaskAnnota
useEffect(() => {
const currentGlobalSelectedIds = useStore.getState().selectedMaskIds;
const validLocalSelectedIds = selectedMaskIds.filter((id) => (
frameMasks.some((mask) => mask.id === id)
));
if (selectedMaskIds.length > 0 && validLocalSelectedIds.length === 0) {
return;
}
if (selectedMaskIds.length === 0) {
const validGlobalSelectedIds = currentGlobalSelectedIds.filter((id) => (
frameMasks.some((mask) => mask.id === id)
));
if (validGlobalSelectedIds.length > 0) return;
}
const isSameSelection = currentGlobalSelectedIds.length === selectedMaskIds.length
&& currentGlobalSelectedIds.every((id, index) => id === selectedMaskIds[index]);
const nextSelectedMaskIds = selectedMaskIds.length > 0 ? validLocalSelectedIds : selectedMaskIds;
const isSameSelection = currentGlobalSelectedIds.length === nextSelectedMaskIds.length
&& currentGlobalSelectedIds.every((id, index) => id === nextSelectedMaskIds[index]);
if (!isSameSelection) {
setGlobalSelectedMaskIds(selectedMaskIds);
setGlobalSelectedMaskIds(nextSelectedMaskIds);
}
}, [frameMasks, selectedMaskIds, setGlobalSelectedMaskIds]);
@@ -468,12 +549,20 @@ export function CanvasArea({ activeTool, frame, onClearMasks, onDeleteMaskAnnota
}
}
if (selectedMaskId && !frameMasks.some((mask) => mask.id === selectedMaskId)) {
setSelectedMaskId(null);
setSelectedMaskIds([]);
const linkedMaskIds = findLinkedMasksOnFrame([selectedMaskId, ...selectedMaskIds], masks, frame?.id);
if (linkedMaskIds.length > 0) {
setSelectedMaskId(linkedMaskIds[0]);
setSelectedMaskIds(linkedMaskIds);
setGlobalSelectedMaskIds(linkedMaskIds);
} else {
setSelectedMaskId(null);
setSelectedMaskIds([]);
setGlobalSelectedMaskIds([]);
}
setSelectedPolygonIndex(0);
setSelectedVertexIndex(null);
}
}, [frameMasks, selectedMaskId]);
}, [frame?.id, frameMasks, masks, selectedMaskId, selectedMaskIds, setGlobalSelectedMaskIds]);
const handleWheel = (e: any) => {
e.evt.preventDefault();

62
src/components/OntologyInspector.test.tsx
View File

@@ -6,11 +6,13 @@ import { OntologyInspector } from './OntologyInspector';
const apiMock = vi.hoisted(() => ({
analyzeMask: vi.fn(),
smoothMaskGeometry: vi.fn(),
updateTemplate: vi.fn(),
}));
vi.mock('../lib/api', () => ({
analyzeMask: apiMock.analyzeMask,
smoothMaskGeometry: apiMock.smoothMaskGeometry,
updateTemplate: apiMock.updateTemplate,
}));
@@ -28,6 +30,17 @@ describe('OntologyInspector', () => {
source: 'sam2.1_hiera_tiny',
message: '已读取后端几何属性',
});
apiMock.smoothMaskGeometry.mockResolvedValue({
polygons: [[[0.12, 0.12], [0.28, 0.12], [0.28, 0.28], [0.12, 0.28]]],
pathData: 'M 12 12 L 28 12 L 28 28 L 12 28 Z',
segmentation: [[12, 12, 28, 12, 28, 28, 12, 28]],
bbox: [12, 12, 16, 16],
area: 256,
topology_anchor_count: 4,
topology_anchors: [],
smoothing: { strength: 35, method: 'chaikin' },
message: '已应用边缘平滑强度 35',
});
useStore.setState({
templates: [
{
@@ -177,6 +190,8 @@ describe('OntologyInspector', () => {
it('loads selected mask properties from the backend analyzer', async () => {
useStore.setState({
frames: [{ id: 'frame-1', projectId: 'p1', index: 0, url: '/1.jpg', width: 100, height: 100 }],
activeClass: { id: 'c3', name: '肿瘤', color: '#f97316', zIndex: 30 },
activeClassId: 'c3',
selectedMaskIds: ['m1'],
masks: [
{
@@ -193,7 +208,11 @@ describe('OntologyInspector', () => {
render(<OntologyInspector />);
expect(await screen.findByText('0.8200')).toBeInTheDocument();
expect(await screen.findByText('4 节点')).toBeInTheDocument();
expect(screen.getAllByText('胆囊')).toHaveLength(2);
expect(screen.queryByText('肿瘤')).not.toBeInTheDocument();
expect(screen.queryByText('后端模型置信度')).not.toBeInTheDocument();
expect(screen.queryByText('0.8200')).not.toBeInTheDocument();
expect(screen.getByText('4 节点')).toBeInTheDocument();
fireEvent.click(screen.getByRole('button', { name: '重新提取拓扑锚点' }));
expect(apiMock.analyzeMask).toHaveBeenLastCalledWith(
@@ -202,4 +221,45 @@ describe('OntologyInspector', () => {
{ extractSkeleton: true },
);
});
it('applies backend edge smoothing to the selected mask and marks it dirty', async () => {
useStore.setState({
frames: [{ id: 'frame-1', projectId: 'p1', index: 0, url: '/1.jpg', width: 100, height: 100 }],
selectedMaskIds: ['m1'],
masks: [
{
id: 'm1',
annotationId: '10',
frameId: 'frame-1',
pathData: 'M 10 10 L 30 10 L 30 30 Z',
label: '胆囊',
color: '#ff0000',
segmentation: [[10, 10, 30, 10, 30, 30]],
saveStatus: 'saved',
saved: true,
},
],
});
render(<OntologyInspector />);
fireEvent.change(screen.getByLabelText('边缘平滑强度'), { target: { value: '35' } });
fireEvent.click(screen.getByRole('button', { name: '应用边缘平滑' }));
await waitFor(() => expect(apiMock.smoothMaskGeometry).toHaveBeenCalledWith(
expect.objectContaining({ id: 'm1' }),
expect.objectContaining({ id: 'frame-1' }),
35,
));
await waitFor(() => expect(useStore.getState().masks[0]).toEqual(expect.objectContaining({
pathData: 'M 12 12 L 28 12 L 28 28 L 12 28 Z',
segmentation: [[12, 12, 28, 12, 28, 28, 12, 28]],
bbox: [12, 12, 16, 16],
area: 256,
saveStatus: 'dirty',
saved: false,
metadata: { geometry_smoothing: { strength: 35, method: 'chaikin' } },
})));
expect(screen.getByText('已应用边缘平滑强度 35，请保存后生效')).toBeInTheDocument();
});
});

102
src/components/OntologyInspector.tsx
View File

@@ -4,7 +4,7 @@ import { useStore } from '../store/useStore';
import type { TemplateClass } from '../store/useStore';
import { cn } from '../lib/utils';
import { getActiveTemplate } from '../lib/templateSelection';
import { analyzeMask, updateTemplate, type MaskAnalysisResult } from '../lib/api';
import { analyzeMask, smoothMaskGeometry, updateTemplate, type MaskAnalysisResult } from '../lib/api';
export function OntologyInspector() {
const templates = useStore((state) => state.templates);
@@ -30,13 +30,17 @@ export function OntologyInspector() {
const [maskAnalysis, setMaskAnalysis] = useState<MaskAnalysisResult | null>(null);
const [isAnalyzingMask, setIsAnalyzingMask] = useState(false);
const [analysisMessage, setAnalysisMessage] = useState('');
const [smoothingStrength, setSmoothingStrength] = useState(0);
const [isSmoothingMask, setIsSmoothingMask] = useState(false);
const activeTemplate = getActiveTemplate(templates, activeTemplateId);
const templateClasses = activeTemplate?.classes || [];
const allClasses = [...templateClasses].sort((a, b) => b.zIndex - a.zIndex);
const selectedMask = masks.find((mask) => selectedMaskIds.includes(mask.id)) || null;
const selectedMaskLabel = selectedMask?.className || selectedMask?.label || '未选择';
const currentFrame = frames[currentFrameIndex] || null;
const classButtonRefs = useRef(new Map<string, HTMLButtonElement>());
const skipNextAutoAnalysisRef = useRef(false);
const selectedMaskClass = useMemo(() => {
if (!selectedMask) return null;
@@ -119,11 +123,68 @@ export function OntologyInspector() {
};
React.useEffect(() => {
if (skipNextAutoAnalysisRef.current) {
skipNextAutoAnalysisRef.current = false;
return;
}
void refreshMaskAnalysis(false);
// selectedMask is intentionally tracked by id and geometry fields to avoid
// re-running analysis for unrelated store changes.
}, [selectedMask?.id, selectedMask?.segmentation, selectedMask?.points, currentFrame?.id]);
React.useEffect(() => {
const smoothing = selectedMask?.metadata?.geometry_smoothing;
const strength = smoothing && typeof smoothing === 'object'
? Number((smoothing as Record<string, unknown>).strength)
: 0;
setSmoothingStrength(Number.isFinite(strength) ? Math.min(Math.max(strength, 0), 100) : 0);
}, [selectedMask?.id]);
const handleApplySmoothing = async () => {
if (!selectedMask || !currentFrame) {
setAnalysisMessage('请选择一个 mask 后再应用边缘平滑');
return;
}
setIsSmoothingMask(true);
setAnalysisMessage('');
try {
const result = await smoothMaskGeometry(selectedMask, currentFrame, smoothingStrength);
skipNextAutoAnalysisRef.current = true;
setMasks(masks.map((mask) => {
if (mask.id !== selectedMask.id) return mask;
return {
...mask,
pathData: result.pathData,
segmentation: result.segmentation,
bbox: result.bbox,
area: result.area,
metadata: {
...(mask.metadata || {}),
geometry_smoothing: result.smoothing,
},
saveStatus: mask.annotationId ? 'dirty' as const : 'draft' as const,
saved: mask.annotationId ? false : mask.saved,
};
}));
setMaskAnalysis({
confidence: null,
confidence_source: 'manual_or_imported',
topology_anchor_count: result.topology_anchor_count,
topology_anchors: result.topology_anchors,
area: result.area,
bbox: result.bbox,
source: selectedMask.metadata?.source as string | undefined,
message: result.message,
});
setAnalysisMessage(`${result.message}，请保存后生效`);
} catch (err) {
console.error('Mask smoothing failed:', err);
setAnalysisMessage('边缘平滑失败，请检查后端服务');
} finally {
setIsSmoothingMask(false);
}
};
const handleAddCustom = async () => {
if (!newClassName.trim()) return;
if (!activeTemplate) {
@@ -301,7 +362,7 @@ export function OntologyInspector() {
<div className="flex items-center gap-2 mb-3">
<Tag size={12} className="text-cyan-400" />
<span className="text-xs font-semibold text-gray-200">
{activeClass?.name || activeTemplate?.name || '未选择'}
{selectedMaskLabel}
</span>
</div>
<div className="space-y-3">
@@ -309,22 +370,35 @@ export function OntologyInspector() {
<span className="text-[10px] text-gray-500 uppercase">当前选中区域:</span>
<span className="text-xs font-mono text-gray-300">{selectedMaskIds.length}</span>
</div>
<div className="space-y-1">
<label className="text-[10px] text-gray-500 uppercase">后端模型置信度</label>
<div className="h-1.5 w-full bg-white/10 rounded-full overflow-hidden">
<div
className="h-full bg-green-500"
style={{ width: `${Math.round((maskAnalysis?.confidence ?? 0) * 100)}%` }}
/>
</div>
<div className="text-[10px] font-mono text-green-500 text-right">
{maskAnalysis?.confidence != null ? maskAnalysis.confidence.toFixed(4) : '无模型分数'}
</div>
</div>
<div className="flex items-center justify-between">
<span className="text-[10px] text-gray-500 uppercase">后端拓扑锚点:</span>
<span className="text-xs font-mono text-gray-300">{maskAnalysis?.topology_anchor_count ?? 0} 节点</span>
</div>
<div>
<div className="mb-2 flex items-center justify-between">
<label htmlFor="mask-edge-smoothing" className="text-[10px] text-gray-500 uppercase">边缘平滑强度:</label>
<span className="text-xs font-mono text-gray-300">{smoothingStrength}%</span>
</div>
<input
id="mask-edge-smoothing"
aria-label="边缘平滑强度"
type="range"
min={0}
max={100}
step={5}
value={smoothingStrength}
onChange={(event) => setSmoothingStrength(Number(event.target.value))}
disabled={!selectedMask || isSmoothingMask}
className="w-full accent-cyan-500 disabled:opacity-40"
/>
<button
onClick={handleApplySmoothing}
disabled={!selectedMask || !currentFrame || isSmoothingMask}
className="mt-2 w-full bg-cyan-500/10 hover:bg-cyan-500/20 border border-cyan-500/20 text-xs text-cyan-100 py-1.5 rounded transition-colors disabled:opacity-40 disabled:cursor-not-allowed"
>
{isSmoothingMask ? '平滑中...' : '应用边缘平滑'}
</button>
</div>
{analysisMessage && (
<div className="text-[10px] leading-relaxed text-gray-500">{analysisMessage}</div>
)}

38
src/components/VideoWorkspace.test.tsx
View File

@@ -233,6 +233,41 @@ describe('VideoWorkspace', () => {
expect(useStore.getState().activeTool).toBe('edit_polygon');
});
it('keeps the current non-first frame when returning from AI segmentation to the workspace', async () => {
apiMock.getProjectFrames.mockResolvedValueOnce([
{ id: 10, project_id: 1, frame_index: 0, image_url: '/frame-1.jpg', width: 640, height: 360 },
{ id: 11, project_id: 1, frame_index: 1, image_url: '/frame-2.jpg', width: 640, height: 360 },
{ id: 12, project_id: 1, frame_index: 2, image_url: '/frame-3.jpg', width: 640, height: 360 },
]);
useStore.setState({
frames: [
{ id: '10', projectId: '1', index: 0, url: '/frame-1.jpg', width: 640, height: 360 },
{ id: '11', projectId: '1', index: 1, url: '/frame-2.jpg', width: 640, height: 360 },
{ id: '12', projectId: '1', index: 2, url: '/frame-3.jpg', width: 640, height: 360 },
],
currentFrameIndex: 1,
activeTool: 'edit_polygon',
selectedMaskIds: ['ai-mask-frame-2'],
masks: [{
id: 'ai-mask-frame-2',
frameId: '11',
pathData: 'M 10 10 L 40 10 L 40 40 Z',
label: 'AI Mask',
color: '#06b6d4',
segmentation: [[10, 10, 40, 10, 40, 40]],
saveStatus: 'draft',
saved: false,
metadata: { source: 'ai_segmentation' },
}],
});
render(<VideoWorkspace />);
await waitFor(() => expect(useStore.getState().frames).toHaveLength(3));
expect(useStore.getState().currentFrameIndex).toBe(1);
expect(useStore.getState().selectedMaskIds).toEqual(['ai-mask-frame-2']);
});
it('saves pending masks through the archive button', async () => {
apiMock.getProjectFrames.mockResolvedValueOnce([
{ id: 10, project_id: 1, frame_index: 0, image_url: '/frame.jpg', width: 640, height: 360 },
@@ -533,6 +568,7 @@ describe('VideoWorkspace', () => {
label: '胆囊',
color: '#ff0000',
class: { id: 'c1', name: '胆囊', color: '#ff0000', zIndex: 20 },
geometry_smoothing: { strength: 35, method: 'chaikin' },
},
bbox: [0.1, 0.1, 0.2, 0.2],
};
@@ -573,6 +609,7 @@ describe('VideoWorkspace', () => {
source_annotation_id: 5,
source_mask_id: 'annotation-5',
propagation_seed_signature: 'seed-signature-5',
geometry_smoothing: { strength: 35, method: 'chaikin' },
},
}],
});
@@ -602,6 +639,7 @@ describe('VideoWorkspace', () => {
template_id: 2,
source_mask_id: 'annotation-5',
source_annotation_id: 5,
smoothing: { strength: 35, method: 'chaikin' },
},
}],
}));

32
src/components/VideoWorkspace.tsx
View File

@@ -159,6 +159,12 @@ export function VideoWorkspace({ onNavigateToAI }: { onNavigateToAI?: () => void
const loadFrames = async () => {
const selectedIdsBeforeLoad = latestSelectedMaskIdsRef.current;
const stateBeforeLoad = useStore.getState();
const selectedFrameIdBeforeLoad = stateBeforeLoad.masks.find((mask) => (
selectedIdsBeforeLoad.includes(mask.id)
&& String(mask.frameId)
))?.frameId;
const currentFrameBeforeLoad = stateBeforeLoad.frames[stateBeforeLoad.currentFrameIndex];
try {
const data = await getProjectFrames(String(currentProject.id));
if (cancelled) return;
@@ -174,8 +180,8 @@ export function VideoWorkspace({ onNavigateToAI }: { onNavigateToAI?: () => void
sourceFrameNumber: f.source_frame_number ?? undefined,
}));
setFrames(mappedFrames);
setCurrentFrame(0);
if (mappedFrames.length === 0) {
setCurrentFrame(0);
setMasks([]);
if (currentProject.status === 'parsing') {
setStatusMessage('生成帧任务正在后台运行，可在 Dashboard 查看进度');
@@ -186,6 +192,16 @@ export function VideoWorkspace({ onNavigateToAI }: { onNavigateToAI?: () => void
}
return;
}
const currentProjectId = String(currentProject.id);
const preferredFrameId = selectedFrameIdBeforeLoad
|| (currentFrameBeforeLoad?.projectId === currentProjectId ? currentFrameBeforeLoad.id : undefined);
const preferredIndex = preferredFrameId
? mappedFrames.findIndex((frame) => frame.id === String(preferredFrameId))
: -1;
const fallbackIndex = currentFrameBeforeLoad?.projectId === currentProjectId
? Math.min(Math.max(stateBeforeLoad.currentFrameIndex, 0), mappedFrames.length - 1)
: 0;
setCurrentFrame(preferredIndex >= 0 ? preferredIndex : fallbackIndex);
setStatusMessage('');
await hydrateSavedAnnotations(String(currentProject.id), mappedFrames, selectedIdsBeforeLoad);
} catch (err) {
@@ -491,6 +507,19 @@ export function VideoWorkspace({ onNavigateToAI }: { onNavigateToAI?: () => void
const inheritedSeedSignature = typeof seedMask.metadata?.propagation_seed_signature === 'string'
? seedMask.metadata.propagation_seed_signature
: undefined;
const rawSmoothing = seedPayload.mask_data?.geometry_smoothing
|| (seedMask.metadata?.geometry_smoothing && typeof seedMask.metadata.geometry_smoothing === 'object'
? seedMask.metadata.geometry_smoothing
: undefined);
const smoothingStrength = rawSmoothing && typeof rawSmoothing === 'object'
? Number((rawSmoothing as Record<string, unknown>).strength)
: NaN;
const geometrySmoothing = Number.isFinite(smoothingStrength) && smoothingStrength > 0
? {
strength: Math.min(Math.max(smoothingStrength, 0), 100),
method: 'chaikin' as const,
}
: undefined;
return {
polygons: seedPayload.mask_data?.polygons,
bbox: seedPayload.bbox,
@@ -502,6 +531,7 @@ export function VideoWorkspace({ onNavigateToAI }: { onNavigateToAI?: () => void
source_mask_id: metadataSourceMaskId || seedMask.id,
source_annotation_id: sourceAnnotationId,
propagation_seed_signature: inheritedSeedSignature,
smoothing: geometrySmoothing,
};
}, [activeTemplateId, currentFrame, currentProject?.id]);

49
src/lib/api.test.ts
View File

@@ -346,6 +346,7 @@ describe('api client contracts', () => {
classZIndex: 20,
segmentation: [[10, 10, 90, 10, 90, 40]],
bbox: [10, 10, 80, 30],
metadata: { geometry_smoothing: { strength: 35, method: 'chaikin' } },
}, frame, '2');
expect(payload).toEqual({
@@ -357,6 +358,7 @@ describe('api client contracts', () => {
label: '胆囊',
color: '#ff0000',
class: { id: 'c1', name: '胆囊', color: '#ff0000', zIndex: 20 },
geometry_smoothing: { strength: 35, method: 'chaikin' },
},
bbox: [0.1, 0.2, 0.8, 0.6],
});
@@ -373,6 +375,7 @@ describe('api client contracts', () => {
class: { id: 'c1', name: '胆囊', color: '#ff0000', zIndex: 20 },
source: 'sam2.1_hiera_tiny_propagation',
propagated_from_frame_id: 4,
geometry_smoothing: { strength: 35, method: 'chaikin' },
},
points: [[0.5, 0.5]],
bbox: null,
@@ -396,10 +399,56 @@ describe('api client contracts', () => {
metadata: {
source: 'sam2.1_hiera_tiny_propagation',
propagated_from_frame_id: 4,
geometry_smoothing: { strength: 35, method: 'chaikin' },
},
}));
});
it('sends selected mask geometry to backend smoothing and maps smoothed polygons back to pixels', async () => {
const { smoothMaskGeometry } = await import('./api');
axiosMock.client.post.mockResolvedValueOnce({
data: {
polygons: [[[0.12, 0.24], [0.88, 0.24], [0.88, 0.76], [0.12, 0.76]]],
topology_anchor_count: 4,
topology_anchors: [[0.12, 0.24]],
area: 0.4,
bbox: [0.12, 0.24, 0.76, 0.52],
smoothing: { strength: 35, method: 'chaikin' },
message: '已应用边缘平滑强度 35',
},
});
const result = await smoothMaskGeometry({
id: 'm1',
frameId: '5',
pathData: 'M 10 10 L 90 10 L 90 40 Z',
label: '胆囊',
color: '#ff0000',
segmentation: [[10, 10, 90, 10, 90, 40]],
bbox: [10, 10, 80, 30],
}, { id: '5', projectId: '9', index: 0, url: '/frame.jpg', width: 100, height: 50 }, 35);
expect(axiosMock.client.post).toHaveBeenCalledWith('/api/ai/smooth-mask', {
frame_id: 5,
mask_data: {
polygons: [[[0.1, 0.2], [0.9, 0.2], [0.9, 0.8]]],
label: '胆囊',
color: '#ff0000',
},
points: undefined,
bbox: [0.1, 0.2, 0.8, 0.6],
strength: 35,
method: 'chaikin',
});
expect(result).toEqual(expect.objectContaining({
pathData: 'M 12 12 L 88 12 L 88 38 L 12 38 Z',
segmentation: [[12, 12, 88, 12, 88, 38, 12, 38]],
bbox: [12, 12, 76, 26],
area: 1976,
smoothing: { strength: 35, method: 'chaikin' },
}));
});
it('preserves editable point regions in annotation payloads', async () => {
const { buildAnnotationPayload } = await import('./api');
const frame = { id: '5', projectId: '9', index: 0, url: '/frame.jpg', width: 100, height: 50 };

88
src/lib/api.ts
View File

@@ -303,6 +303,7 @@ export interface SavedAnnotation {
source?: string;
propagated_from_frame_id?: number;
propagated_from_frame_index?: number;
geometry_smoothing?: GeometrySmoothingOptions;
score?: number | null;
[key: string]: unknown;
} | null;
@@ -327,6 +328,8 @@ export interface SaveAnnotationPayload {
zIndex?: number;
category?: string;
};
geometry_smoothing?: GeometrySmoothingOptions;
[key: string]: unknown;
};
points?: number[][];
bbox?: number[];
@@ -354,6 +357,8 @@ export interface PropagateMasksPayload {
template_id?: number;
source_mask_id?: string;
source_annotation_id?: number;
propagation_seed_signature?: string;
smoothing?: GeometrySmoothingOptions;
};
direction?: 'forward' | 'backward' | 'both';
max_frames?: number;
@@ -394,6 +399,23 @@ export interface MaskAnalysisResult {
message: string;
}
export interface GeometrySmoothingOptions {
strength: number;
method: 'chaikin';
}
export interface SmoothMaskGeometryResult {
polygons: number[][][];
pathData: string;
segmentation: number[][];
bbox: [number, number, number, number];
area: number;
topology_anchor_count: number;
topology_anchors: number[][];
smoothing: GeometrySmoothingOptions;
message: string;
}
export interface DashboardTask {
id: string;
task_id?: number;
@@ -461,6 +483,28 @@ function polygonToBbox(points: number[][], width: number, height: number): [numb
return [minX, minY, maxX - minX, maxY - minY];
}
function polygonAreaPixels(points: number[][], width: number, height: number): number {
if (points.length < 3) return 0;
let total = 0;
points.forEach(([x, y], index) => {
const [nx, ny] = points[(index + 1) % points.length];
total += (x * width) * (ny * height);
total -= (nx * width) * (y * height);
});
return Math.abs(total) / 2;
}
function normalizeGeometrySmoothing(value: unknown): GeometrySmoothingOptions | undefined {
if (!value || typeof value !== 'object') return undefined;
const source = value as Record<string, unknown>;
const strength = Number(source.strength);
if (!Number.isFinite(strength) || strength <= 0) return undefined;
return {
strength: Math.min(Math.max(strength, 0), 100),
method: 'chaikin',
};
}
function pixelSegmentationToNormalizedPolygons(
segmentation: number[][] | undefined,
width: number,
@@ -498,6 +542,7 @@ export function buildAnnotationPayload(
zIndex: mask.classZIndex,
}
: undefined;
const geometrySmoothing = normalizeGeometrySmoothing(mask.metadata?.geometry_smoothing);
const payload: SaveAnnotationPayload = {
project_id: Number(projectId),
@@ -508,6 +553,7 @@ export function buildAnnotationPayload(
label: mask.label,
color: mask.color,
...(classMetadata ? { class: classMetadata } : {}),
...(geometrySmoothing ? { geometry_smoothing: geometrySmoothing } : {}),
},
bbox: mask.bbox
? [
@@ -587,6 +633,48 @@ export async function analyzeMask(mask: Mask, frame: Frame, options: { extractSk
return response.data;
}
export async function smoothMaskGeometry(mask: Mask, frame: Frame, strength: number): Promise<SmoothMaskGeometryResult> {
const polygons = pixelSegmentationToNormalizedPolygons(mask.segmentation, frame.width, frame.height);
const response = await apiClient.post('/api/ai/smooth-mask', {
frame_id: Number(frame.id),
mask_data: {
polygons,
label: mask.label,
color: mask.color,
},
points: mask.points?.map(([x, y]) => [
clamp01(x / Math.max(frame.width, 1)),
clamp01(y / Math.max(frame.height, 1)),
]),
bbox: mask.bbox
? [
clamp01(mask.bbox[0] / Math.max(frame.width, 1)),
clamp01(mask.bbox[1] / Math.max(frame.height, 1)),
clamp01(mask.bbox[2] / Math.max(frame.width, 1)),
clamp01(mask.bbox[3] / Math.max(frame.height, 1)),
]
: undefined,
strength,
method: 'chaikin',
});
const resultPolygons: number[][][] = response.data.polygons || [];
const firstPolygon = resultPolygons[0] || [];
const bbox = firstPolygon.length > 0
? polygonToBbox(firstPolygon, frame.width, frame.height)
: [0, 0, 0, 0] as [number, number, number, number];
return {
polygons: resultPolygons,
pathData: firstPolygon.length > 0 ? polygonToPath(firstPolygon, frame.width, frame.height) : '',
segmentation: resultPolygons.map((polygon) => polygon.flatMap(([x, y]) => [x * frame.width, y * frame.height])),
bbox,
area: resultPolygons.reduce((total, polygon) => total + polygonAreaPixels(polygon, frame.width, frame.height), 0),
topology_anchor_count: response.data.topology_anchor_count,
topology_anchors: response.data.topology_anchors || [],
smoothing: response.data.smoothing,
message: response.data.message,
};
}
export async function predictMask(payload: PredictMaskPayload): Promise<PredictMaskResult> {
let prompt_type: 'point' | 'box' | 'semantic' | 'interactive';
let prompt_data: unknown;