import React, { useEffect, useRef, useState, useCallback } from 'react'; import { Stage, Layer, Image as KonvaImage, Circle, Rect, Path, Group } from 'react-konva'; import polygonClipping, { type MultiPolygon, type Pair } from 'polygon-clipping'; import useImage from 'use-image'; import { useStore } from '../store/useStore'; import { predictMask } from '../lib/api'; import type { Frame, Mask } from '../store/useStore'; interface CanvasAreaProps { activeTool: string; frame: Frame | null; onDeleteMaskAnnotations?: (annotationIds: string[]) => Promise | void; } type CanvasPoint = { x: number; y: number }; type PromptPoint = CanvasPoint & { type: 'pos' | 'neg' }; type PromptBox = { x1: number; y1: number; x2: number; y2: number }; type ToolHint = { title: string; body: string }; const DRAG_MANUAL_TOOLS = new Set(['create_rectangle', 'create_circle']); const POLYGON_TOOL = 'create_polygon'; const EDIT_POLYGON_TOOL = 'edit_polygon'; const BRUSH_TOOL = 'brush'; const ERASER_TOOL = 'eraser'; const PAINT_TOOLS = new Set([BRUSH_TOOL, ERASER_TOOL]); const BOOLEAN_TOOLS = new Set(['area_merge', 'area_remove']); const POLYGON_CLOSE_RADIUS = 8; const DEFAULT_IMAGE_FIT_RATIO = 0.86; const TOOL_HINT_TTL_MS = 3600; const PAINT_STAMP_SEGMENTS = 16; const MAX_PAINT_STROKE_POINTS = 128; function clamp(value: number, min: number, max: number): number { return Math.min(Math.max(value, min), max); } function metadataNumber(value: unknown): number | null { const parsed = Number(value); return Number.isFinite(parsed) && parsed > 0 ? parsed : null; } function propagationSourceMaskTokens(value: unknown): string[] { if (typeof value !== 'string' || value.length === 0) return []; const tokens = [`mask:${value}`]; const annotationMatch = value.match(/^annotation-(\d+)$/); if (annotationMatch) { tokens.push(`annotation:${annotationMatch[1]}`); } return tokens; } function isPropagationMask(mask: Mask): boolean { const metadata = mask.metadata || {}; const source = typeof metadata.source === 'string' ? metadata.source : ''; return source.includes('_propagation') || metadata.propagated_from_frame_id !== undefined || metadata.propagation_seed_key !== undefined || metadata.source_annotation_id !== undefined || metadata.source_mask_id !== undefined; } function propagationLineageTokens(mask: Mask): Set { const metadata = mask.metadata || {}; const tokens = new Set([`mask:${mask.id}`]); if (mask.annotationId) { tokens.add(`annotation:${mask.annotationId}`); } const sourceAnnotationId = metadataNumber(metadata.source_annotation_id); if (sourceAnnotationId !== null) { tokens.add(`annotation:${sourceAnnotationId}`); } propagationSourceMaskTokens(metadata.source_mask_id).forEach((token) => tokens.add(token)); if (typeof metadata.propagation_seed_key === 'string' && metadata.propagation_seed_key.length > 0) { tokens.add(`seed-key:${metadata.propagation_seed_key}`); } return tokens; } function findLinkedMasksOnFrame(selectedIds: string[], allMasks: Mask[], targetFrameId?: string): string[] { if (!targetFrameId || selectedIds.length === 0) return []; const selectedMasks = selectedIds .map((id) => allMasks.find((mask) => mask.id === id)) .filter((mask): mask is Mask => Boolean(mask)); if (selectedMasks.length === 0) return []; const selectedTokens = new Set(); const selectedHasPropagation = selectedMasks.some(isPropagationMask); selectedMasks.forEach((mask) => { propagationLineageTokens(mask).forEach((token) => selectedTokens.add(token)); }); return allMasks .filter((mask) => String(mask.frameId) === String(targetFrameId)) .filter((mask) => { const candidateHasPropagation = isPropagationMask(mask); if (!selectedHasPropagation && !candidateHasPropagation) return false; const candidateTokens = propagationLineageTokens(mask); return [...candidateTokens].some((token) => selectedTokens.has(token)); }) .map((mask) => mask.id); } function findPropagationChainMaskIds(selectedIds: string[], allMasks: Mask[]): Set { const selectedMasks = selectedIds .map((id) => allMasks.find((mask) => mask.id === id)) .filter((mask): mask is Mask => Boolean(mask)); const selectedTokens = new Set(); selectedMasks.forEach((mask) => { propagationLineageTokens(mask).forEach((token) => selectedTokens.add(token)); }); if (selectedTokens.size === 0) return new Set(selectedIds); return new Set( allMasks .filter((mask) => { const candidateTokens = propagationLineageTokens(mask); return [...candidateTokens].some((token) => selectedTokens.has(token)); }) .map((mask) => mask.id), ); } function expandedPropagationDeletionMaskIds(selectedIds: string[], allMasks: Mask[]): Set { const selectedIdSet = new Set(selectedIds); const chainIds = findPropagationChainMaskIds(selectedIds, allMasks); return new Set( allMasks .filter((mask) => selectedIdSet.has(mask.id) || (chainIds.has(mask.id) && isPropagationMask(mask))) .map((mask) => mask.id), ); } function maskLayerPriority(mask: Mask): number { const parsed = Number(mask.classZIndex ?? mask.metadata?.classZIndex ?? 0); return Number.isFinite(parsed) ? parsed : 0; } function polygonPath(points: CanvasPoint[]): string { if (points.length === 0) return ''; return points .map((point, index) => `${index === 0 ? 'M' : 'L'} ${point.x} ${point.y}`) .join(' ') .concat(' Z'); } function segmentationPath(segmentation?: number[][]): string { return (segmentation || []) .map((polygon) => polygonPath(flatPolygonToPoints(polygon))) .filter(Boolean) .join(' '); } function segmentationPolygonPath(segmentation: number[][] | undefined, polygonIndex: number): string { const polygon = segmentation?.[polygonIndex]; return polygon ? polygonPath(flatPolygonToPoints(polygon)) : ''; } function polygonSegmentation(points: CanvasPoint[]): number[][] { return [points.flatMap((point) => [point.x, point.y])]; } function segmentationToPoints(segmentation?: number[][], polygonIndex = 0): CanvasPoint[] { const polygon = segmentation?.[polygonIndex] || []; const points: CanvasPoint[] = []; for (let index = 0; index < polygon.length - 1; index += 2) { points.push({ x: polygon[index], y: polygon[index + 1] }); } return points; } function flatPolygonToPoints(polygon: number[]): CanvasPoint[] { const points: CanvasPoint[] = []; for (let index = 0; index < polygon.length - 1; index += 2) { points.push({ x: polygon[index], y: polygon[index + 1] }); } return points; } function segmentationAllPoints(segmentation?: number[][]): CanvasPoint[] { return (segmentation || []).flatMap((polygon) => flatPolygonToPoints(polygon)); } function polygonBbox(points: CanvasPoint[]): [number, number, number, number] { const xs = points.map((point) => point.x); const ys = points.map((point) => point.y); const minX = Math.min(...xs); const minY = Math.min(...ys); const maxX = Math.max(...xs); const maxY = Math.max(...ys); return [minX, minY, maxX - minX, maxY - minY]; } function polygonArea(points: CanvasPoint[]): number { if (points.length < 3) return 0; const sum = points.reduce((acc, point, index) => { const next = points[(index + 1) % points.length]; return acc + point.x * next.y - next.x * point.y; }, 0); return Math.abs(sum) / 2; } function pointDistance(a: CanvasPoint, b: CanvasPoint): number { return Math.hypot(a.x - b.x, a.y - b.y); } function extendStrokePoints( current: CanvasPoint[], nextPoint: CanvasPoint, spacing: number, maxPoints = MAX_PAINT_STROKE_POINTS, ): CanvasPoint[] { const previous = current[current.length - 1]; if (!previous) return [nextPoint]; const distance = pointDistance(previous, nextPoint); if (distance < spacing) return current; const steps = Math.max(1, Math.floor(distance / spacing)); const additions: CanvasPoint[] = []; for (let step = 1; step <= steps; step += 1) { if (current.length + additions.length >= maxPoints) break; const ratio = step / steps; additions.push({ x: previous.x + (nextPoint.x - previous.x) * ratio, y: previous.y + (nextPoint.y - previous.y) * ratio, }); } return [...current, ...additions]; } function distanceToSegmentSquared(point: CanvasPoint, start: CanvasPoint, end: CanvasPoint): number { const dx = end.x - start.x; const dy = end.y - start.y; const lengthSquared = dx * dx + dy * dy; if (lengthSquared === 0) { return (point.x - start.x) ** 2 + (point.y - start.y) ** 2; } const t = clamp(((point.x - start.x) * dx + (point.y - start.y) * dy) / lengthSquared, 0, 1); const projected = { x: start.x + t * dx, y: start.y + t * dy }; return (point.x - projected.x) ** 2 + (point.y - projected.y) ** 2; } function nearestPolygonEdgeIndex(points: CanvasPoint[], point: CanvasPoint): number { return points.reduce((bestIndex, start, index) => { const end = points[(index + 1) % points.length]; if (!end) return bestIndex; const bestStart = points[bestIndex]; const bestEnd = points[(bestIndex + 1) % points.length]; const currentDistance = distanceToSegmentSquared(point, start, end); const bestDistance = bestStart && bestEnd ? distanceToSegmentSquared(point, bestStart, bestEnd) : Number.POSITIVE_INFINITY; return currentDistance < bestDistance ? index : bestIndex; }, 0); } function segmentationArea(segmentation?: number[][]): number { return (segmentation || []).reduce((sum, polygon) => sum + polygonArea(flatPolygonToPoints(polygon)), 0); } function segmentationBbox(segmentation?: number[][]): [number, number, number, number] | undefined { const points = segmentationAllPoints(segmentation); return points.length > 0 ? polygonBbox(points) : undefined; } function closeRing(points: CanvasPoint[]): Pair[] { const ring = points.map((point) => [point.x, point.y] as Pair); const first = ring[0]; const last = ring[ring.length - 1]; if (first && last && (first[0] !== last[0] || first[1] !== last[1])) { ring.push([first[0], first[1]]); } return ring; } function segmentationRings(segmentation?: number[][]): Pair[][] { return (segmentation || []) .map((polygon) => closeRing(flatPolygonToPoints(polygon))) .filter((ring) => ring.length >= 4); } function maskPolygonRingCounts(mask: Mask, ringCount: number): number[] | null { const rawCounts = mask.metadata?.polygonRingCounts; if (!Array.isArray(rawCounts)) return null; const counts = rawCounts .map((count) => Number(count)) .filter((count) => Number.isInteger(count) && count > 0); const total = counts.reduce((sum, count) => sum + count, 0); return total === ringCount ? counts : null; } function maskToMultiPolygon(mask: Mask): MultiPolygon | null { const rings = segmentationRings(mask.segmentation); if (rings.length === 0) return null; const counts = maskPolygonRingCounts(mask, rings.length); if (counts) { let offset = 0; return counts.map((count) => { const polygon = rings.slice(offset, offset + count); offset += count; return polygon; }).filter((polygon) => polygon.length > 0); } if (mask.metadata?.hasHoles && rings.length > 1) { return [rings]; } return rings.map((ring) => [ring]); } function polygonsToMultiPolygon(polygons: CanvasPoint[][]): MultiPolygon | null { const geometry = polygons .filter((points) => points.length >= 3) .map((points) => [closeRing(points)]); return geometry.length > 0 ? geometry : null; } function openRingPoints(ring: Pair[]): CanvasPoint[] { const openRing = ring.length > 1 && ring[0][0] === ring[ring.length - 1][0] && ring[0][1] === ring[ring.length - 1][1] ? ring.slice(0, -1) : ring; return openRing.map(([x, y]) => ({ x, y })); } function multiPolygonToSegmentation(geometry: MultiPolygon): number[][] { return geometry .flatMap((polygon) => polygon) .map((ring) => openRingPoints(ring).flatMap(({ x, y }) => [x, y])) .filter((polygon) => polygon.length >= 6); } function multiPolygonRingCounts(geometry: MultiPolygon): number[] { return geometry .map((polygon) => polygon.length) .filter((count) => count > 0); } function multiPolygonArea(geometry: MultiPolygon): number { return geometry.reduce((sum, polygon) => { const [outerRing, ...holeRings] = polygon; const outerArea = outerRing ? polygonArea(openRingPoints(outerRing)) : 0; const holesArea = holeRings.reduce((holeSum, ring) => holeSum + polygonArea(openRingPoints(ring)), 0); return sum + Math.max(outerArea - holesArea, 0); }, 0); } function multiPolygonHasHoles(geometry: MultiPolygon): boolean { return geometry.some((polygon) => polygon.length > 1); } function maskWithSegmentation( mask: Mask, segmentation: number[][], options: { area?: number; hasHoles?: boolean; polygonRingCounts?: number[] } = {}, ): Mask { const bbox = segmentationBbox(segmentation); const metadata = { ...(mask.metadata || {}) }; if (options.hasHoles === true) metadata.hasHoles = true; if (options.hasHoles === false) delete metadata.hasHoles; if (options.polygonRingCounts && options.polygonRingCounts.length > 0) metadata.polygonRingCounts = options.polygonRingCounts; if (options.hasHoles === false) delete metadata.polygonRingCounts; return { ...mask, pathData: segmentationPath(segmentation), segmentation, bbox, area: options.area ?? segmentationArea(segmentation), metadata, saveStatus: mask.annotationId ? 'dirty' : 'draft', saved: mask.annotationId ? false : mask.saved, }; } function rectanglePoints(start: CanvasPoint, end: CanvasPoint): CanvasPoint[] { const x1 = Math.min(start.x, end.x); const y1 = Math.min(start.y, end.y); const x2 = Math.max(start.x, end.x); const y2 = Math.max(start.y, end.y); return [ { x: x1, y: y1 }, { x: x2, y: y1 }, { x: x2, y: y2 }, { x: x1, y: y2 }, ]; } function circlePoints(start: CanvasPoint, end: CanvasPoint): CanvasPoint[] { const cx = (start.x + end.x) / 2; const cy = (start.y + end.y) / 2; const rx = Math.abs(end.x - start.x) / 2; const ry = Math.abs(end.y - start.y) / 2; return Array.from({ length: 32 }, (_, index) => { const angle = (Math.PI * 2 * index) / 32; return { x: cx + Math.cos(angle) * rx, y: cy + Math.sin(angle) * ry }; }); } function circleStampPoints(center: CanvasPoint, radius: number, segments = PAINT_STAMP_SEGMENTS): CanvasPoint[] { return Array.from({ length: segments }, (_, index) => { const angle = (Math.PI * 2 * index) / segments; return { x: center.x + Math.cos(angle) * radius, y: center.y + Math.sin(angle) * radius }; }); } function paintStrokeToGeometry(strokePoints: CanvasPoint[], radius: number): MultiPolygon | null { const geometries = strokePoints .map((point) => polygonsToMultiPolygon([circleStampPoints(point, radius)])) .filter((geometry): geometry is MultiPolygon => Boolean(geometry)); if (geometries.length === 0) return null; const [firstGeometry, ...restGeometries] = geometries; return restGeometries.length === 0 ? firstGeometry : polygonClipping.union(firstGeometry, ...restGeometries); } function geometriesOverlap(first: MultiPolygon, second: MultiPolygon): boolean { return polygonClipping.intersection(first, second).length > 0; } export function CanvasArea({ activeTool, frame, onDeleteMaskAnnotations }: CanvasAreaProps) { const containerRef = useRef(null); const [stageSize, setStageSize] = useState({ width: 800, height: 600 }); const [scale, setScale] = useState(1); const [position, setPosition] = useState({ x: 0, y: 0 }); const [points, setPoints] = useState([]); const [cursorPos, setCursorPos] = useState({ x: 0, y: 0 }); const [boxStart, setBoxStart] = useState<{ x: number, y: number } | null>(null); const [boxCurrent, setBoxCurrent] = useState<{ x: number, y: number } | null>(null); const [samPromptBox, setSamPromptBox] = useState(null); const [samCandidateMaskId, setSamCandidateMaskId] = useState(null); const [manualStart, setManualStart] = useState(null); const [manualCurrent, setManualCurrent] = useState(null); const [paintStrokePoints, setPaintStrokePointsState] = useState([]); const [polygonPoints, setPolygonPoints] = useState([]); const [selectedMaskId, setSelectedMaskId] = useState(() => useStore.getState().selectedMaskIds[0] || null); const [selectedMaskIds, setSelectedMaskIds] = useState(() => useStore.getState().selectedMaskIds); const [selectedPolygonIndex, setSelectedPolygonIndex] = useState(0); const [selectedVertexIndex, setSelectedVertexIndex] = useState(null); const previousFrameIdRef = useRef(frame?.id); const [isInferencing, setIsInferencing] = useState(false); const [inferenceMessage, setInferenceMessage] = useState(''); const [isToolHintVisible, setIsToolHintVisible] = useState(false); const lastAutoFitKeyRef = useRef(''); const paintStrokeRef = useRef([]); const paintToolRef = useRef(null); const lastPaintPointRef = useRef(null); const masks = useStore((state) => state.masks); const addMask = useStore((state) => state.addMask); const updateMask = useStore((state) => state.updateMask); const setMasks = useStore((state) => state.setMasks); const setGlobalSelectedMaskIds = useStore((state) => state.setSelectedMaskIds); const maskPreviewOpacity = useStore((state) => state.maskPreviewOpacity); const brushSize = useStore((state) => state.brushSize); const eraserSize = useStore((state) => state.eraserSize); const storeActiveTool = useStore((state) => state.activeTool); const aiModel = useStore((state) => state.aiModel); const activeTemplateId = useStore((state) => state.activeTemplateId); const activeClass = useStore((state) => state.activeClass); const effectiveTool = activeTool || storeActiveTool; // Load the actual frame image const [image] = useImage(frame?.url || ''); const frameMasks = masks.filter((mask) => mask.frameId === frame?.id); const displayFrameMasks = React.useMemo(() => { if (selectedMaskIds.length > 0) return frameMasks; return frameMasks .map((mask, index) => ({ mask, index })) .sort((a, b) => { const priorityDiff = maskLayerPriority(a.mask) - maskLayerPriority(b.mask); return priorityDiff === 0 ? a.index - b.index : priorityDiff; }) .map((item) => item.mask); }, [frameMasks, selectedMaskIds.length]); const selectedMask = React.useMemo( () => frameMasks.find((mask) => mask.id === selectedMaskId) || null, [frameMasks, selectedMaskId], ); const booleanSelectedMasks = React.useMemo( () => selectedMaskIds .map((id) => frameMasks.find((mask) => mask.id === id)) .filter((mask): mask is Mask => Boolean(mask)), [frameMasks, selectedMaskIds], ); const selectedMaskPoints = React.useMemo( () => segmentationToPoints(selectedMask?.segmentation, selectedPolygonIndex), [selectedMask?.segmentation, selectedPolygonIndex], ); const savedMaskCount = frameMasks.filter((mask) => mask.saveStatus === 'saved' || mask.saved).length; const draftMaskCount = frameMasks.filter((mask) => !mask.annotationId).length; const dirtyMaskCount = frameMasks.filter((mask) => mask.saveStatus === 'dirty').length; const isBooleanTool = BOOLEAN_TOOLS.has(effectiveTool); const isPaintTool = PAINT_TOOLS.has(effectiveTool); const isPolygonEditTool = effectiveTool === 'move' || effectiveTool === EDIT_POLYGON_TOOL; const activePaintSize = effectiveTool === ERASER_TOOL ? eraserSize : brushSize; const activePaintRadius = Math.max(2, activePaintSize / 2); const setPaintStrokePoints = useCallback((nextPoints: CanvasPoint[]) => { paintStrokeRef.current = nextPoints; setPaintStrokePointsState(nextPoints); }, []); const currentLayerLabel = selectedMask ? `${selectedMask.className || selectedMask.label}${selectedMask.annotationId ? ` #${selectedMask.annotationId}` : ' (未保存)'}` : '未选择'; const toolHint = React.useMemo(() => { if (!frame) return null; if (effectiveTool === POLYGON_TOOL) { if (polygonPoints.length === 0) { return { title: '创建多边形', body: '点击画布添加顶点;至少 3 个点后,点击首节点或按 Enter 完成,按 Esc 取消。', }; } if (polygonPoints.length < 3) { return { title: `创建多边形 · 已放置 ${polygonPoints.length} 点`, body: '继续点击添加顶点;满 3 个点后才能闭合,按 Esc 可取消当前多边形。', }; } return { title: `创建多边形 · 已放置 ${polygonPoints.length} 点`, body: '点击黄色首节点或按 Enter 闭合完成;按 Esc 放弃当前多边形。', }; } if (effectiveTool === 'create_rectangle') { return { title: '创建矩形', body: '按住并拖拽框出区域,松开鼠标后生成 mask;切换工具可放弃当前操作。' }; } if (effectiveTool === 'create_circle') { return { title: '创建圆形', body: '按住并拖拽确定外接范围,松开鼠标后生成椭圆 mask。' }; } if (effectiveTool === BRUSH_TOOL) { return { title: '画笔', body: activeClass ? '按住并拖动画出连续区域;若与当前选中 mask 连通,会自动合并到该 mask。' : '先在右侧语义分类树选择类别,然后按住并拖动画出连续区域。', }; } if (effectiveTool === ERASER_TOOL) { return { title: '橡皮擦', body: selectedMask ? '按住并拖动,从当前选中 mask 中扣除经过的区域。' : '先选择一个 mask,然后按住并拖动擦除区域。', }; } if (effectiveTool === 'box_select') { return { title: samPromptBox ? '边界框已建立' : '边界框选', body: samPromptBox ? '继续添加正向/反向点可细化同一个候选区域;重新拖拽会替换当前框。' : '按住并拖拽建立框选区域,松开后会触发 SAM 推理。', }; } if (effectiveTool === 'point_pos') { return { title: '正向选点', body: '点击目标内部添加正向点并触发细化;点击已有提示点可删除并重新推理。' }; } if (effectiveTool === 'point_neg') { return { title: '反向选点', body: '点击不应包含的区域添加反向点;点击已有提示点可删除并重新推理。' }; } if (effectiveTool === 'area_merge') { return { title: '区域合并', body: booleanSelectedMasks.length > 0 ? `已选 ${booleanSelectedMasks.length} 个区域;第一个选中的是主区域,点击“合并选中”完成。` : '依次点击多个 mask;第一个选中的区域会作为合并后的主区域。', }; } if (effectiveTool === 'area_remove') { return { title: '重叠区域去除', body: booleanSelectedMasks.length > 0 ? `已选 ${booleanSelectedMasks.length} 个区域;第一个是保留主区域,后续区域会被扣除。` : '先点击要保留的主区域,再点击要扣除的干涉区域。', }; } if (effectiveTool === EDIT_POLYGON_TOOL || (effectiveTool === 'move' && selectedMask)) { return { title: selectedMask ? '调整多边形' : '调整多边形', body: selectedMask ? '可直接拖动白色顶点;点击青色边中点或双击边线新增顶点;选中顶点/区域后按 Delete 删除。' : '点击一个 mask 后,可拖动顶点、点击边中点新增顶点,或按 Delete 删除选中区域。', }; } return null; }, [activeClass, booleanSelectedMasks.length, effectiveTool, frame, polygonPoints.length, samPromptBox, selectedMask]); useEffect(() => { if (!toolHint) { setIsToolHintVisible(false); return; } setIsToolHintVisible(true); const timer = window.setTimeout(() => { setIsToolHintVisible(false); }, TOOL_HINT_TTL_MS); return () => window.clearTimeout(timer); }, [toolHint?.body, toolHint?.title]); useEffect(() => { const handleResize = () => { if (containerRef.current) { setStageSize({ width: containerRef.current.clientWidth, height: containerRef.current.clientHeight, }); } }; handleResize(); window.addEventListener('resize', handleResize); return () => window.removeEventListener('resize', handleResize); }, []); useEffect(() => { if (!frame?.id || stageSize.width <= 0 || stageSize.height <= 0) return; const imageWidth = frame.width || image?.naturalWidth || image?.width || 0; const imageHeight = frame.height || image?.naturalHeight || image?.height || 0; if (imageWidth <= 0 || imageHeight <= 0) return; const fitKey = `${frame.id}:${stageSize.width}x${stageSize.height}:${imageWidth}x${imageHeight}`; if (lastAutoFitKeyRef.current === fitKey) return; lastAutoFitKeyRef.current = fitKey; const nextScale = Math.max( 0.05, Math.min(stageSize.width / imageWidth, stageSize.height / imageHeight) * DEFAULT_IMAGE_FIT_RATIO, ); setScale(nextScale); setPosition({ x: (stageSize.width - imageWidth * nextScale) / 2, y: (stageSize.height - imageHeight * nextScale) / 2, }); }, [frame?.height, frame?.id, frame?.width, image?.height, image?.naturalHeight, image?.naturalWidth, image?.width, stageSize.height, stageSize.width]); useEffect(() => { setManualStart(null); setManualCurrent(null); setPaintStrokePoints([]); paintToolRef.current = null; lastPaintPointRef.current = null; setPolygonPoints([]); setSelectedVertexIndex(null); if (!isPolygonEditTool && !isBooleanTool && !isPaintTool) { setSelectedMaskId(null); setSelectedMaskIds([]); setSelectedPolygonIndex(0); } }, [effectiveTool, isBooleanTool, isPaintTool, isPolygonEditTool, setPaintStrokePoints]); useEffect(() => { if (previousFrameIdRef.current === frame?.id) return; previousFrameIdRef.current = frame?.id; const linkedMaskIds = findLinkedMasksOnFrame(useStore.getState().selectedMaskIds, masks, frame?.id); if (linkedMaskIds.length > 0) { setSelectedMaskId(linkedMaskIds[0]); setSelectedMaskIds(linkedMaskIds); setGlobalSelectedMaskIds(linkedMaskIds); } else { setSelectedMaskId(null); setSelectedMaskIds([]); setGlobalSelectedMaskIds([]); } setSelectedPolygonIndex(0); setSelectedVertexIndex(null); }, [frame?.id, masks, setGlobalSelectedMaskIds]); useEffect(() => { setPoints([]); setSamPromptBox(null); setSamCandidateMaskId(null); }, [frame?.id]); useEffect(() => { const currentGlobalSelectedIds = useStore.getState().selectedMaskIds; const validLocalSelectedIds = selectedMaskIds.filter((id) => ( frameMasks.some((mask) => mask.id === id) )); if (selectedMaskIds.length > 0 && validLocalSelectedIds.length === 0) { return; } if (selectedMaskIds.length === 0) { const validGlobalSelectedIds = currentGlobalSelectedIds.filter((id) => ( frameMasks.some((mask) => mask.id === id) )); if (validGlobalSelectedIds.length > 0) return; } const nextSelectedMaskIds = selectedMaskIds.length > 0 ? validLocalSelectedIds : selectedMaskIds; const isSameSelection = currentGlobalSelectedIds.length === nextSelectedMaskIds.length && currentGlobalSelectedIds.every((id, index) => id === nextSelectedMaskIds[index]); if (!isSameSelection) { setGlobalSelectedMaskIds(nextSelectedMaskIds); } }, [frameMasks, selectedMaskIds, setGlobalSelectedMaskIds]); useEffect(() => () => setGlobalSelectedMaskIds([]), [setGlobalSelectedMaskIds]); useEffect(() => { if (!selectedMaskId) { const validGlobalSelectedIds = useStore.getState().selectedMaskIds.filter((id) => ( frameMasks.some((mask) => mask.id === id) )); if (validGlobalSelectedIds.length > 0) { setSelectedMaskId(validGlobalSelectedIds[0]); setSelectedMaskIds(validGlobalSelectedIds); setSelectedPolygonIndex(0); setSelectedVertexIndex(null); return; } } if (selectedMaskId && !frameMasks.some((mask) => mask.id === selectedMaskId)) { const linkedMaskIds = findLinkedMasksOnFrame([selectedMaskId, ...selectedMaskIds], masks, frame?.id); if (linkedMaskIds.length > 0) { setSelectedMaskId(linkedMaskIds[0]); setSelectedMaskIds(linkedMaskIds); setGlobalSelectedMaskIds(linkedMaskIds); } else { setSelectedMaskId(null); setSelectedMaskIds([]); setGlobalSelectedMaskIds([]); } setSelectedPolygonIndex(0); setSelectedVertexIndex(null); } }, [frame?.id, frameMasks, masks, selectedMaskId, selectedMaskIds, setGlobalSelectedMaskIds]); const handleWheel = (e: any) => { e.evt.preventDefault(); const scaleBy = 1.1; const stage = e.target.getStage(); const oldScale = stage.scaleX(); const mousePointTo = { x: stage.getPointerPosition().x / oldScale - stage.x() / oldScale, y: stage.getPointerPosition().y / oldScale - stage.y() / oldScale, }; const newScale = e.evt.deltaY < 0 ? oldScale * scaleBy : oldScale / scaleBy; setScale(newScale); setPosition({ x: -(mousePointTo.x - stage.getPointerPosition().x / newScale) * newScale, y: -(mousePointTo.y - stage.getPointerPosition().y / newScale) * newScale, }); }; const handleStageDragEnd = (e: any) => { const stage = e.target?.getStage?.(); if (!stage || e.target !== stage) return; setPosition({ x: stage.x(), y: stage.y(), }); }; const stagePoint = (e: any): CanvasPoint | null => { const stage = e.target.getStage(); const relPos = stage?.getRelativePointerPosition(); if (!relPos) return null; const imageWidth = frame?.width || image?.naturalWidth || image?.width || stageSize.width; const imageHeight = frame?.height || image?.naturalHeight || image?.height || stageSize.height; return { x: clamp(relPos.x, 0, imageWidth), y: clamp(relPos.y, 0, imageHeight), }; }; const createManualMask = useCallback((shape: string, polygon: CanvasPoint[]) => { if (!frame?.id || polygon.length < 3) return; const area = polygonArea(polygon); if (area <= 1) return; const color = activeClass?.color || '#06b6d4'; const label = activeClass?.name || `手工${shape}`; const mask: Mask = { id: `manual-${frame.id}-${shape}-${Date.now()}`, frameId: frame.id, templateId: activeTemplateId || undefined, classId: activeClass?.id, className: activeClass?.name, classZIndex: activeClass?.zIndex, classMaskId: activeClass?.maskId, saveStatus: 'draft', saved: false, pathData: polygonPath(polygon), label, color, segmentation: polygonSegmentation(polygon), bbox: polygonBbox(polygon), area, metadata: { source: 'manual', shape }, }; addMask(mask); }, [activeClass, activeTemplateId, addMask, frame?.id]); const createManualMaskFromGeometry = useCallback((shape: string, geometry: MultiPolygon): Mask | null => { if (!frame?.id || !activeClass) return null; const segmentation = multiPolygonToSegmentation(geometry); const polygonRingCounts = multiPolygonRingCounts(geometry); if (segmentation.length === 0) return null; const area = multiPolygonArea(geometry); if (area <= 1) return null; const mask: Mask = { id: `manual-${frame.id}-${shape}-${Date.now()}`, frameId: frame.id, templateId: activeTemplateId || undefined, classId: activeClass.id, className: activeClass.name, classZIndex: activeClass.zIndex, classMaskId: activeClass.maskId, saveStatus: 'draft', saved: false, pathData: segmentationPath(segmentation), label: activeClass.name, color: activeClass.color, segmentation, bbox: segmentationBbox(segmentation), area, metadata: { source: 'manual', shape, ...(multiPolygonHasHoles(geometry) ? { hasHoles: true } : {}), ...(multiPolygonHasHoles(geometry) ? { polygonRingCounts } : {}), }, }; addMask(mask); return mask; }, [activeClass, activeTemplateId, addMask, frame?.id]); const finishPolygon = useCallback(() => { if (polygonPoints.length < 3) return; createManualMask('多边形', polygonPoints); setPolygonPoints([]); }, [createManualMask, polygonPoints]); const handleMouseMove = (e: any) => { const stage = e.target.getStage(); if (!stage) return; const pos = stage.getPointerPosition(); if (pos) { const imageX = (pos.x - position.x) / scale; const imageY = (pos.y - position.y) / scale; setCursorPos({ x: imageX, y: imageY }); } if (boxStart && effectiveTool === 'box_select') { const relPos = stage.getRelativePointerPosition(); if (relPos) { setBoxCurrent({ x: relPos.x, y: relPos.y }); } } if (manualStart && DRAG_MANUAL_TOOLS.has(effectiveTool)) { const pos = stage.getRelativePointerPosition(); if (pos) { setManualCurrent({ x: pos.x, y: pos.y }); } } if (paintToolRef.current && PAINT_TOOLS.has(effectiveTool)) { const pos = stagePoint(e); const previous = lastPaintPointRef.current; if (!pos || !previous) return; const radius = Math.max(2, (paintToolRef.current === ERASER_TOOL ? eraserSize : brushSize) / 2); const minDistance = Math.max(3, radius * 0.55); if (pointDistance(previous, pos) < minDistance) return; const currentStroke = paintStrokeRef.current; if (currentStroke.length >= MAX_PAINT_STROKE_POINTS) return; const nextStroke = extendStrokePoints(currentStroke, pos, minDistance); lastPaintPointRef.current = nextStroke[nextStroke.length - 1] || pos; setPaintStrokePoints(nextStroke); } }; const runInference = useCallback(async ( promptPoints?: PromptPoint[], promptBox?: PromptBox, options: { resetCandidate?: boolean } = {}, ) => { if (!frame?.id) { console.warn('Inference skipped: no active frame'); setInferenceMessage('请先选择一帧图像。'); return; } const imageWidth = frame.width || image?.naturalWidth || image?.width || 0; const imageHeight = frame.height || image?.naturalHeight || image?.height || 0; if (imageWidth <= 0 || imageHeight <= 0) { console.warn('Inference skipped: active frame dimensions are unavailable'); setInferenceMessage('当前帧缺少宽高信息,无法推理。'); return; } setIsInferencing(true); setInferenceMessage(''); try { const hasNegativePrompt = Boolean(promptPoints?.some((point) => point.type === 'neg')); const existingCandidate = !options.resetCandidate && samCandidateMaskId ? masks.find((mask) => mask.id === samCandidateMaskId) : null; const result = await predictMask({ imageId: frame.id, imageWidth, imageHeight, model: aiModel, points: promptPoints && promptPoints.length > 0 ? promptPoints.map((p) => ({ x: p.x, y: p.y, type: p.type })) : undefined, box: promptBox, ...(hasNegativePrompt ? { options: { auto_filter_background: true, min_score: 0.05 } } : {}), }); const [m] = result.masks; if (m) { const label = activeClass?.name || existingCandidate?.label || m.label; const color = activeClass?.color || existingCandidate?.color || m.color; const metadata = { ...(existingCandidate?.metadata || {}), source: 'sam2_interactive', promptBox: promptBox || null, promptPointCount: promptPoints?.length || 0, promptNegativePointCount: promptPoints?.filter((point) => point.type === 'neg').length || 0, }; const nextMask = { frameId: frame.id, templateId: activeTemplateId || existingCandidate?.templateId || undefined, classId: activeClass?.id || existingCandidate?.classId, className: activeClass?.name || existingCandidate?.className, classZIndex: activeClass?.zIndex ?? existingCandidate?.classZIndex, classMaskId: activeClass?.maskId ?? existingCandidate?.classMaskId, saveStatus: existingCandidate?.annotationId ? 'dirty' as const : 'draft' as const, saved: false, pathData: m.pathData, label, color, segmentation: m.segmentation, points: promptPoints?.filter((p) => p.type === 'pos').map((p) => [p.x, p.y]), bbox: m.bbox, area: m.area, metadata, }; if (existingCandidate) { updateMask(existingCandidate.id, nextMask); setSelectedMaskId(existingCandidate.id); setSelectedMaskIds([existingCandidate.id]); } else { const id = m.id; setSamCandidateMaskId(id); setSelectedMaskId(id); setSelectedMaskIds([id]); addMask({ id, ...nextMask, }); } } else { if (existingCandidate && hasNegativePrompt) { setMasks(masks.filter((mask) => mask.id !== existingCandidate.id)); setSamCandidateMaskId(null); setSelectedMaskId(null); setSelectedMaskIds([]); setInferenceMessage('反向点已排除当前候选区域,请重新框选或添加新的正向点。'); } else { setInferenceMessage('模型没有返回可用区域,请调整点/框提示后重试。'); } } } catch (err) { console.error('Inference failed:', err); const detail = (err as any)?.response?.data?.detail; setInferenceMessage(detail || 'AI 推理失败,请查看模型状态或后端日志。'); } finally { setIsInferencing(false); } }, [activeClass, activeTemplateId, addMask, aiModel, frame?.height, frame?.id, frame?.width, image?.height, image?.naturalHeight, image?.naturalWidth, image?.width, masks, samCandidateMaskId, setMasks, updateMask]); const deleteMasksById = useCallback((maskIds: string[]) => { if (maskIds.length === 0) return; const idSet = expandedPropagationDeletionMaskIds(maskIds, masks); const deletingMasks = masks.filter((mask) => idSet.has(mask.id)); if (deletingMasks.length === 0) return; setMasks(masks.filter((mask) => !idSet.has(mask.id))); const annotationIds = deletingMasks .map((mask) => mask.annotationId) .filter((annotationId): annotationId is string => Boolean(annotationId)); if (annotationIds.length > 0) { void onDeleteMaskAnnotations?.(annotationIds); } if (samCandidateMaskId && idSet.has(samCandidateMaskId)) { setSamCandidateMaskId(null); setSamPromptBox(null); setPoints([]); } setSelectedMaskId(null); setSelectedMaskIds([]); setSelectedPolygonIndex(0); setSelectedVertexIndex(null); }, [masks, onDeleteMaskAnnotations, samCandidateMaskId, setMasks]); const applyPaintStroke = useCallback((tool: string | null, strokePoints: CanvasPoint[]) => { if (!frame?.id || strokePoints.length === 0) return; const radius = Math.max(2, (tool === ERASER_TOOL ? eraserSize : brushSize) / 2); const strokeGeometry = paintStrokeToGeometry(strokePoints, radius); if (!strokeGeometry) return; if (tool === BRUSH_TOOL) { if (!activeClass) { setInferenceMessage('请先在右侧语义分类树选择类别,再使用画笔。'); return; } const targetGeometry = selectedMask ? maskToMultiPolygon(selectedMask) : null; const shouldMerge = Boolean(targetGeometry && geometriesOverlap(targetGeometry, strokeGeometry)); if (selectedMask && targetGeometry && shouldMerge) { const resultGeometry = polygonClipping.union(targetGeometry, strokeGeometry); const resultSegmentation = multiPolygonToSegmentation(resultGeometry); if (resultSegmentation.length === 0) return; const nextMask = { ...maskWithSegmentation(selectedMask, resultSegmentation, { area: multiPolygonArea(resultGeometry), hasHoles: multiPolygonHasHoles(resultGeometry), polygonRingCounts: multiPolygonRingCounts(resultGeometry), }), templateId: activeTemplateId || selectedMask.templateId, classId: activeClass.id, className: activeClass.name, classZIndex: activeClass.zIndex, classMaskId: activeClass.maskId, label: activeClass.name, color: activeClass.color, }; setMasks(masks.map((mask) => (mask.id === selectedMask.id ? nextMask : mask))); setSelectedMaskId(selectedMask.id); setSelectedMaskIds([selectedMask.id]); setSelectedVertexIndex(null); return; } const nextMask = createManualMaskFromGeometry('画笔', strokeGeometry); if (nextMask) { setSelectedMaskId(nextMask.id); setSelectedMaskIds([nextMask.id]); setSelectedPolygonIndex(0); setSelectedVertexIndex(null); } return; } if (tool === ERASER_TOOL) { if (!selectedMask) { setInferenceMessage('请先选择一个 mask,再使用橡皮擦。'); return; } const targetGeometry = maskToMultiPolygon(selectedMask); if (!targetGeometry) return; const resultGeometry = polygonClipping.difference(targetGeometry, strokeGeometry); const resultSegmentation = multiPolygonToSegmentation(resultGeometry); if (resultSegmentation.length === 0) { deleteMasksById([selectedMask.id]); return; } const nextMask = maskWithSegmentation(selectedMask, resultSegmentation, { area: multiPolygonArea(resultGeometry), hasHoles: multiPolygonHasHoles(resultGeometry), polygonRingCounts: multiPolygonRingCounts(resultGeometry), }); setMasks(masks.map((mask) => (mask.id === selectedMask.id ? nextMask : mask))); setSelectedMaskId(selectedMask.id); setSelectedMaskIds([selectedMask.id]); setSelectedVertexIndex(null); } }, [ activeClass, activeTemplateId, brushSize, createManualMaskFromGeometry, deleteMasksById, eraserSize, frame?.id, masks, selectedMask, setMasks, ]); const handleStageMouseDown = (e: any) => { if (PAINT_TOOLS.has(effectiveTool)) { const canStart = effectiveTool === BRUSH_TOOL ? Boolean(activeClass) : Boolean(selectedMask); if (!canStart) return; const pos = stagePoint(e); if (pos) { paintToolRef.current = effectiveTool; lastPaintPointRef.current = pos; setPaintStrokePoints([pos]); } return; } if (DRAG_MANUAL_TOOLS.has(effectiveTool)) { const pos = stagePoint(e); if (pos) { setManualStart(pos); setManualCurrent(pos); } return; } if (effectiveTool === 'box_select') { const stage = e.target.getStage(); const pos = stage.getRelativePointerPosition(); if (pos) { setBoxStart({ x: pos.x, y: pos.y }); setBoxCurrent({ x: pos.x, y: pos.y }); } } }; const handleStageMouseUp = (e: any) => { if (paintToolRef.current && PAINT_TOOLS.has(effectiveTool)) { const finalPoint = stagePoint(e); const currentStroke = paintStrokeRef.current; const spacing = Math.max(3, activePaintRadius * 0.55); const nextStroke = finalPoint && currentStroke.length > 0 && pointDistance(currentStroke[currentStroke.length - 1], finalPoint) >= spacing && currentStroke.length < MAX_PAINT_STROKE_POINTS ? extendStrokePoints(currentStroke, finalPoint, spacing) : currentStroke; const tool = paintToolRef.current; setPaintStrokePoints([]); paintToolRef.current = null; lastPaintPointRef.current = null; applyPaintStroke(tool, nextStroke); return; } if (DRAG_MANUAL_TOOLS.has(effectiveTool) && manualStart) { const end = stagePoint(e) || manualCurrent || manualStart; const width = Math.abs(end.x - manualStart.x); const height = Math.abs(end.y - manualStart.y); if (effectiveTool === 'create_rectangle' && width > 4 && height > 4) { createManualMask('矩形', rectanglePoints(manualStart, end)); } if (effectiveTool === 'create_circle' && width > 4 && height > 4) { createManualMask('圆形', circlePoints(manualStart, end)); } setManualStart(null); setManualCurrent(null); return; } if (effectiveTool === 'box_select' && boxStart && boxCurrent) { const x1 = Math.min(boxStart.x, boxCurrent.x); const y1 = Math.min(boxStart.y, boxCurrent.y); const x2 = Math.max(boxStart.x, boxCurrent.x); const y2 = Math.max(boxStart.y, boxCurrent.y); if (Math.abs(x2 - x1) > 5 && Math.abs(y2 - y1) > 5) { const nextBox = { x1, y1, x2, y2 }; setPoints([]); setSamPromptBox(nextBox); setSamCandidateMaskId(null); runInference([], nextBox, { resetCandidate: true }); } setBoxStart(null); setBoxCurrent(null); } }; const handleStageClick = (e: any) => { if (isPolygonEditTool) return; if (effectiveTool === 'box_select') return; // handled by mouseup if (DRAG_MANUAL_TOOLS.has(effectiveTool)) return; if (PAINT_TOOLS.has(effectiveTool)) return; if (effectiveTool === POLYGON_TOOL) { const pos = stagePoint(e); if (pos) { const closeRadius = POLYGON_CLOSE_RADIUS / Math.max(scale, 0.1); if (polygonPoints.length >= 3 && pointDistance(pos, polygonPoints[0]) <= closeRadius) { finishPolygon(); return; } setPolygonPoints((current) => [...current, pos]); } return; } if (effectiveTool === 'point_pos' || effectiveTool === 'point_neg') { const stage = e.target.getStage(); const pos = stage.getRelativePointerPosition(); if (pos) { const newPoints = [ ...points, { x: pos.x, y: pos.y, type: (effectiveTool === 'point_pos' ? 'pos' : 'neg') as 'pos' | 'neg' }, ]; setPoints(newPoints); runInference(newPoints, samPromptBox || undefined); } } }; const removePromptPoint = useCallback((pointIndex: number, event?: any) => { if (event) event.cancelBubble = true; const nextPoints = points.filter((_, index) => index !== pointIndex); setPoints(nextPoints); if (nextPoints.length > 0 || samPromptBox) { runInference(nextPoints, samPromptBox || undefined); return; } if (samCandidateMaskId) { setMasks(masks.filter((mask) => mask.id !== samCandidateMaskId)); setSamCandidateMaskId(null); setSelectedMaskId(null); setSelectedMaskIds([]); setInferenceMessage('已移除最后一个提示点和对应候选区域。'); } }, [masks, points, runInference, samCandidateMaskId, samPromptBox, setMasks]); const updatePolygonMask = useCallback((mask: Mask, nextPoints: CanvasPoint[], polygonIndex = 0) => { if (nextPoints.length < 3) return; const nextSegmentation = [...(mask.segmentation || [])]; nextSegmentation[polygonIndex] = nextPoints.flatMap((point) => [point.x, point.y]); const bbox = segmentationBbox(nextSegmentation) || polygonBbox(nextPoints); const nextGeometry = maskToMultiPolygon({ ...mask, segmentation: nextSegmentation }); const hasHoles = nextGeometry ? multiPolygonHasHoles(nextGeometry) : Boolean(mask.metadata?.hasHoles); const metadata = { ...(mask.metadata || {}) }; if (hasHoles && nextGeometry) { metadata.hasHoles = true; metadata.polygonRingCounts = multiPolygonRingCounts(nextGeometry); } else { delete metadata.hasHoles; delete metadata.polygonRingCounts; } updateMask(mask.id, { pathData: segmentationPath(nextSegmentation), segmentation: nextSegmentation, bbox, area: nextGeometry ? multiPolygonArea(nextGeometry) : segmentationArea(nextSegmentation), metadata, saveStatus: mask.annotationId ? 'dirty' : 'draft', saved: mask.annotationId ? false : mask.saved, }); }, [updateMask]); const updateMaskFromSegmentation = useCallback(( mask: Mask, segmentation: number[][], options: { area?: number; hasHoles?: boolean; polygonRingCounts?: number[] } = {}, ): Mask => { return maskWithSegmentation(mask, segmentation, options); }, []); useEffect(() => { const handleKeyDown = (event: KeyboardEvent) => { if ((event.key === 'Delete' || event.key === 'Backspace') && selectedMask && selectedVertexIndex !== null) { const currentPoints = segmentationToPoints(selectedMask.segmentation, selectedPolygonIndex); if (currentPoints.length > 3) { event.preventDefault(); const nextPoints = currentPoints.filter((_, index) => index !== selectedVertexIndex); updatePolygonMask(selectedMask, nextPoints, selectedPolygonIndex); setSelectedVertexIndex(null); } return; } if ((event.key === 'Delete' || event.key === 'Backspace') && selectedMask) { event.preventDefault(); const ids = selectedMaskIds.length > 0 ? selectedMaskIds : [selectedMask.id]; deleteMasksById(ids); return; } if (effectiveTool !== POLYGON_TOOL) return; if (event.key === 'Enter' && polygonPoints.length >= 3) { event.preventDefault(); finishPolygon(); } if (event.key === 'Escape') { event.preventDefault(); setPolygonPoints([]); } }; window.addEventListener('keydown', handleKeyDown); return () => window.removeEventListener('keydown', handleKeyDown); }, [deleteMasksById, effectiveTool, finishPolygon, isPolygonEditTool, polygonPoints, selectedMask, selectedMaskIds, selectedPolygonIndex, selectedVertexIndex, updatePolygonMask]); const boxRect = React.useMemo(() => { if (!boxStart || !boxCurrent) return null; const x = Math.min(boxStart.x, boxCurrent.x); const y = Math.min(boxStart.y, boxCurrent.y); const width = Math.abs(boxCurrent.x - boxStart.x); const height = Math.abs(boxCurrent.y - boxStart.y); return { x, y, width, height }; }, [boxStart, boxCurrent]); const manualPreviewPath = React.useMemo(() => { if (manualStart && manualCurrent) { if (effectiveTool === 'create_rectangle') return polygonPath(rectanglePoints(manualStart, manualCurrent)); if (effectiveTool === 'create_circle') return polygonPath(circlePoints(manualStart, manualCurrent)); } if (effectiveTool === POLYGON_TOOL && polygonPoints.length > 0) { const previewPoints = [...polygonPoints, cursorPos]; return polygonPath(previewPoints); } return null; }, [cursorPos, effectiveTool, manualCurrent, manualStart, polygonPoints]); const selectedMaskEditableRings = React.useMemo(() => { if (!selectedMask?.segmentation) return []; const hasHoles = Boolean(selectedMask.metadata?.hasHoles); if (!hasHoles) { return [{ polygonIndex: selectedPolygonIndex, points: selectedMaskPoints }]; } return selectedMask.segmentation .map((_, polygonIndex) => ({ polygonIndex, points: segmentationToPoints(selectedMask.segmentation, polygonIndex) })) .filter((ring) => ring.points.length >= 3); }, [selectedMask, selectedMaskPoints, selectedPolygonIndex]); const handleMaskSelect = (mask: Mask, event: any, polygonIndex = 0) => { if (!isPolygonEditTool && !isBooleanTool) return; event.cancelBubble = true; if (isBooleanTool) { setSelectedMaskIds((current) => ( current.includes(mask.id) ? current.filter((id) => id !== mask.id) : [...current, mask.id] )); setSelectedMaskId(mask.id); setSelectedPolygonIndex(polygonIndex); setSelectedVertexIndex(null); return; } setSelectedMaskId(mask.id); setSelectedMaskIds([mask.id]); setSelectedPolygonIndex(polygonIndex); setSelectedVertexIndex(null); }; const handleVertexDragStart = (mask: Mask, vertexIndex: number, polygonIndex = selectedPolygonIndex, event?: any) => { if (event) event.cancelBubble = true; setSelectedMaskId(mask.id); setSelectedMaskIds([mask.id]); setSelectedPolygonIndex(polygonIndex); setSelectedVertexIndex(vertexIndex); }; const handleVertexDrag = (mask: Mask, vertexIndex: number, event: any, polygonIndex = selectedPolygonIndex) => { const imageWidth = frame?.width || image?.naturalWidth || image?.width || stageSize.width; const imageHeight = frame?.height || image?.naturalHeight || image?.height || stageSize.height; const currentPoints = segmentationToPoints(mask.segmentation, polygonIndex); if (!currentPoints[vertexIndex]) return; const nextPoints = currentPoints.map((point, index) => ( index === vertexIndex ? { x: clamp(event.target.x(), 0, imageWidth), y: clamp(event.target.y(), 0, imageHeight), } : point )); setSelectedMaskId(mask.id); setSelectedMaskIds([mask.id]); setSelectedPolygonIndex(polygonIndex); setSelectedVertexIndex(vertexIndex); updatePolygonMask(mask, nextPoints, polygonIndex); }; const handleEdgeInsert = (mask: Mask, edgeIndex: number, event: any, polygonIndex = selectedPolygonIndex) => { event.cancelBubble = true; const currentPoints = segmentationToPoints(mask.segmentation, polygonIndex); const start = currentPoints[edgeIndex]; const end = currentPoints[(edgeIndex + 1) % currentPoints.length]; if (!start || !end) return; const inserted = { x: (start.x + end.x) / 2, y: (start.y + end.y) / 2 }; const nextPoints = [ ...currentPoints.slice(0, edgeIndex + 1), inserted, ...currentPoints.slice(edgeIndex + 1), ]; setSelectedMaskId(mask.id); setSelectedPolygonIndex(polygonIndex); setSelectedVertexIndex(edgeIndex + 1); updatePolygonMask(mask, nextPoints, polygonIndex); }; const handlePathDoubleClick = (mask: Mask, event: any, polygonIndex = 0) => { if (effectiveTool !== EDIT_POLYGON_TOOL) return; event.cancelBubble = true; const point = stagePoint(event); const currentPoints = segmentationToPoints(mask.segmentation, polygonIndex); if (!point || currentPoints.length < 3) return; const edgeIndex = nearestPolygonEdgeIndex(currentPoints, point); const nextPoints = [ ...currentPoints.slice(0, edgeIndex + 1), point, ...currentPoints.slice(edgeIndex + 1), ]; setSelectedMaskId(mask.id); setSelectedMaskIds([mask.id]); setSelectedPolygonIndex(polygonIndex); setSelectedVertexIndex(edgeIndex + 1); updatePolygonMask(mask, nextPoints, polygonIndex); }; const handleBooleanOperation = async () => { if (!frame || booleanSelectedMasks.length < 2) return; const primary = booleanSelectedMasks[0]; const secondaryMasks = booleanSelectedMasks.slice(1); const currentFrameId = String(frame.id); const targetFrameIds = new Set([currentFrameId]); masks.forEach((mask) => { const targetFrameId = String(mask.frameId); if (targetFrameId === currentFrameId) return; const hasPrimary = findLinkedMasksOnFrame([primary.id], masks, targetFrameId).length > 0; if (!hasPrimary) return; const hasSecondary = secondaryMasks.some((secondary) => ( findLinkedMasksOnFrame([secondary.id], masks, targetFrameId).length > 0 )); if (hasSecondary) targetFrameIds.add(targetFrameId); }); const updatedMasks = new Map(); const deletedMaskIds = new Set(); const applyOperationForFrame = (targetFrameId: string) => { const primaryTargetId = targetFrameId === currentFrameId ? primary.id : findLinkedMasksOnFrame([primary.id], masks, targetFrameId)[0]; const primaryTarget = masks.find((mask) => mask.id === primaryTargetId); if (!primaryTarget || deletedMaskIds.has(primaryTarget.id)) return; const primaryGeometry = maskToMultiPolygon(primaryTarget); if (!primaryGeometry) return; const secondaryTargetIds = Array.from(new Set( secondaryMasks.flatMap((secondary) => ( targetFrameId === currentFrameId ? [secondary.id] : findLinkedMasksOnFrame([secondary.id], masks, targetFrameId) )), )).filter((maskId) => maskId !== primaryTarget.id && !deletedMaskIds.has(maskId)); const secondaryTargets = secondaryTargetIds .map((maskId) => masks.find((mask) => mask.id === maskId)) .filter((mask): mask is Mask => Boolean(mask)); const clipGeometries = secondaryTargets .map(maskToMultiPolygon) .filter((geometry): geometry is MultiPolygon => Boolean(geometry)); if (clipGeometries.length === 0) return; const resultGeometry = effectiveTool === 'area_merge' ? polygonClipping.union(primaryGeometry, ...clipGeometries) : polygonClipping.difference(primaryGeometry, ...clipGeometries); const resultSegmentation = multiPolygonToSegmentation(resultGeometry); if (resultSegmentation.length === 0) { deletedMaskIds.add(primaryTarget.id); } else { updatedMasks.set(primaryTarget.id, updateMaskFromSegmentation(primaryTarget, resultSegmentation, { area: multiPolygonArea(resultGeometry), hasHoles: multiPolygonHasHoles(resultGeometry), polygonRingCounts: multiPolygonRingCounts(resultGeometry), })); } if (effectiveTool === 'area_merge') { secondaryTargets.forEach((mask) => deletedMaskIds.add(mask.id)); } }; targetFrameIds.forEach(applyOperationForFrame); const deletedAnnotationIds = Array.from(new Set( masks .filter((mask) => deletedMaskIds.has(mask.id)) .map((mask) => mask.annotationId) .filter((annotationId): annotationId is string => Boolean(annotationId)), )); setMasks(masks .filter((mask) => !deletedMaskIds.has(mask.id)) .map((mask) => updatedMasks.get(mask.id) || mask)); if (deletedAnnotationIds.length > 0) await onDeleteMaskAnnotations?.(deletedAnnotationIds); if (deletedMaskIds.has(primary.id)) { setSelectedMaskId(null); setSelectedMaskIds([]); } else { setSelectedMaskId(primary.id); setSelectedMaskIds([primary.id]); } setSelectedVertexIndex(null); }; return (
{isInferencing && (
AI 推理中...
)} {!isInferencing && inferenceMessage && (
{inferenceMessage}
)} {toolHint && isToolHintVisible && (
{toolHint.title}
{toolHint.body}
)} {/* Background Image Layer */} {image && ( )} {/* AI Returned Masks */} {displayFrameMasks.map((mask) => { const selectedIndex = selectedMaskIds.indexOf(mask.id); const isMaskSelected = selectedIndex >= 0; const isBooleanPrimary = isBooleanTool && selectedIndex === 0; const isBooleanSecondary = isBooleanTool && selectedIndex > 0; const strokeColor = isBooleanPrimary ? '#facc15' : isBooleanSecondary ? '#fb7185' : mask.color; const strokeDash = isBooleanSecondary ? [6 / scale, 4 / scale] : undefined; const hasHoles = Boolean(mask.metadata?.hasHoles); const paths = hasHoles ? [{ data: segmentationPath(mask.segmentation), polygonIndex: 0, fillRule: 'evenodd' }] : (mask.segmentation && mask.segmentation.length > 0 ? mask.segmentation : [undefined]).map((_, polygonIndex) => ({ data: mask.segmentation ? segmentationPolygonPath(mask.segmentation, polygonIndex) : mask.pathData, polygonIndex, fillRule: undefined, })); return ( {paths.map(({ data, polygonIndex, fillRule }) => ( handleMaskSelect(mask, event, polygonIndex)} onTap={(event: any) => handleMaskSelect(mask, event, polygonIndex)} onDblClick={(event: any) => handlePathDoubleClick(mask, event, polygonIndex)} onDblTap={(event: any) => handlePathDoubleClick(mask, event, polygonIndex)} /> ))} ); })} {/* Box selection preview */} {boxRect && effectiveTool === 'box_select' && ( )} {/* Manual shape preview */} {manualPreviewPath && ( )} {paintStrokePoints.length > 0 && ( {paintStrokePoints.map((point, index) => ( ))} )} {isPaintTool && (effectiveTool === BRUSH_TOOL ? activeClass : selectedMask) && paintStrokePoints.length === 0 && ( )} {polygonPoints.map((point, index) => ( = 3 ? 6 : 4) / scale} fill={index === 0 && polygonPoints.length >= 3 ? '#facc15' : '#22d3ee'} stroke={index === 0 && polygonPoints.length >= 3 ? '#fef3c7' : '#ffffff'} strokeWidth={1 / scale} onClick={(event: any) => { if (index !== 0 || polygonPoints.length < 3) return; event.cancelBubble = true; finishPolygon(); }} onTap={(event: any) => { if (index !== 0 || polygonPoints.length < 3) return; event.cancelBubble = true; finishPolygon(); }} /> ))} {/* Polygon edge insertion handles */} {isPolygonEditTool && selectedMask && selectedMaskEditableRings.flatMap(({ polygonIndex, points: ringPoints }) => ( ringPoints.map((point, index) => { const next = ringPoints[(index + 1) % ringPoints.length]; if (!next) return null; return ( handleEdgeInsert(selectedMask, index, event, polygonIndex)} onTap={(event: any) => handleEdgeInsert(selectedMask, index, event, polygonIndex)} /> ); }) ))} {/* Polygon vertex editor */} {isPolygonEditTool && selectedMask && selectedMaskEditableRings.flatMap(({ polygonIndex, points: ringPoints }) => ( ringPoints.map((point, index) => { const isActiveVertex = selectedPolygonIndex === polygonIndex && selectedVertexIndex === index; return ( handleVertexDragStart(selectedMask, index, polygonIndex, event)} onTouchStart={(event: any) => handleVertexDragStart(selectedMask, index, polygonIndex, event)} onDragStart={(event: any) => handleVertexDragStart(selectedMask, index, polygonIndex, event)} onClick={(event: any) => { event.cancelBubble = true; setSelectedPolygonIndex(polygonIndex); setSelectedVertexIndex(index); }} onTap={(event: any) => { event.cancelBubble = true; setSelectedPolygonIndex(polygonIndex); setSelectedVertexIndex(index); }} onDragMove={(event: any) => handleVertexDrag(selectedMask, index, event, polygonIndex)} onDragEnd={(event: any) => handleVertexDrag(selectedMask, index, event, polygonIndex)} /> ); }) ))} {/* AI Prompts Point Regions */} {points.map((p, i) => ( removePromptPoint(i, event)} onTap={(event: any) => removePromptPoint(i, event)} /> removePromptPoint(i, event)} onTap={(event: any) => removePromptPoint(i, event)} /> ))}
光标: {cursorPos.x.toFixed(2)}, {cursorPos.y.toFixed(2)} 当前图层: {currentLayerLabel} 缩放比: {(scale * 100).toFixed(0)}% 遮罩数: {frameMasks.length} 已保存: {savedMaskCount} 未保存: {draftMaskCount} 待更新: {dirtyMaskCount}
{frameMasks.length > 0 && isBooleanTool && (
已选 {booleanSelectedMasks.length}
)}
); }