Files
Pre_Seg_Server/src/lib/api.ts
admin 5ab4602535 feat: 完善视频传播、标注编辑和拆帧闭环
- 接入 SAM2 视频传播能力:新增 /api/ai/propagate,支持用当前帧 mask/polygon/bbox 作为 seed,通过 SAM2 video predictor 向前、向后或双向传播,并可保存为真实 annotation。
- 接入 SAM3 video tracker:通过独立 Python 3.12 external worker 调用 SAM3 video predictor/tracker,使用本地 checkpoint 与 bbox seed 执行视频级跟踪,并在模型状态中标记 video_track 能力。
- 完善 SAM 模型分发:sam_registry 按 model_id 明确区分 sam2 propagation 与 sam3 video_track,避免两个模型链路混用。
- 打通前端“传播片段”:VideoWorkspace 使用当前选中 mask 和当前 AI 模型调用后端传播接口,传播结果回写并刷新工作区已保存标注。
- 增强 SAM3 本地 checkpoint 配置:新增 sam3_checkpoint_path 配置和 .env.example 示例,状态检查改为基于本地 checkpoint/独立环境/模型包可用性。
- 完善视频拆帧参数:/api/media/parse 支持 parse_fps、max_frames、target_width,后端任务保存帧时间戳、源帧号和 frame_sequence 元数据。
- 增加运行时 schema 兼容处理:启动时为旧 frames 表补充 timestamp_ms 和 source_frame_number 列,避免旧库升级后缺字段。
- 强化 Canvas 标注编辑:补齐多边形闭合、点工具、顶点拖拽、边中点插入、Delete/Backspace 删除、区域合并和重叠去除等交互。
- 增强语义分类联动:选中 mask 后可通过右侧语义分类树更新标签、颜色和 class metadata,并同步到保存/导出链路。
- 增加关键帧时间轴体验:FrameTimeline 显示具体时间信息,并支持键盘左右方向键切换关键帧。
- 完善 AI 交互分割参数:前端保留正向点、反向点、框选和 interactive prompt 的调用状态,支持 SAM2 细化候选区域与 SAM3 bbox 入口。
- 扩展后端/前端 API 类型:新增 propagateMasks、传播请求/响应 schema,并补齐 annotation、导出、模型状态和任务接口的测试覆盖。
- 更新项目文档:同步 README、AGENTS、接口契约、需求冻结、设计冻结、前端元素审计、实施计划和测试计划,标明真实功能边界与剩余风险。
- 增加测试覆盖:补充 SAM2/SAM3 传播、SAM3 状态、媒体拆帧参数、Canvas 编辑、语义标签切换、时间轴、工作区传播和 API 合约测试。
- 加强仓库安全边界:将 sam3权重/ 加入 .gitignore,避免本地模型权重被误提交。

验证:npm run test:run;pytest backend/tests;npm run lint;npm run build;python -m py_compile;git diff --check。
2026-05-01 20:27:33 +08:00

659 lines
19 KiB
TypeScript

import axios, { AxiosError } from 'axios';
import type { AiModelId, Frame, Mask, Project, Template } from '../store/useStore';
import { API_BASE_URL } from './config';
const apiClient = axios.create({
baseURL: API_BASE_URL,
headers: {
'Content-Type': 'application/json',
},
timeout: 30000,
});
// Request interceptor: attach token
apiClient.interceptors.request.use(
(config) => {
const token = localStorage.getItem('token');
if (token) {
config.headers.Authorization = `Bearer ${token}`;
}
return config;
},
(error) => Promise.reject(error)
);
// Response interceptor: handle errors
apiClient.interceptors.response.use(
(response) => response,
(error: AxiosError) => {
if (error.response?.status === 401) {
localStorage.removeItem('token');
window.location.reload();
}
return Promise.reject(error);
}
);
// Auth
export async function login(username: string, password: string): Promise<{ token: string }> {
const response = await apiClient.post('/api/auth/login', { username, password });
return response.data;
}
// Projects
function normalizeProjectStatus(status?: string): Project['status'] {
const value = (status || 'pending').toLowerCase();
if (value === 'ready') return 'ready';
if (value === 'parsing' || value === 'queued' || value === 'running') return 'parsing';
if (value === 'error' || value === 'failed') return 'error';
return 'pending';
}
function mapProject(p: any): Project {
return {
id: String(p.id),
name: p.name,
description: p.description,
status: normalizeProjectStatus(p.status),
frames: p.frame_count ?? 0,
fps: p.original_fps ? `${Math.round(p.original_fps)}FPS` : '30FPS',
thumbnail_url: p.thumbnail_url,
video_path: p.video_path,
source_type: p.source_type,
original_fps: p.original_fps,
parse_fps: p.parse_fps,
createdAt: p.created_at,
updatedAt: p.updated_at,
};
}
export async function getProjects(): Promise<Project[]> {
const response = await apiClient.get('/api/projects');
return response.data.map(mapProject);
}
export async function createProject(payload: {
name: string;
description?: string;
parse_fps?: number;
}): Promise<Project> {
const response = await apiClient.post('/api/projects', payload);
return mapProject(response.data);
}
export async function updateProject(id: string, payload: Partial<Project>): Promise<Project> {
const response = await apiClient.patch(`/api/projects/${id}`, payload);
return mapProject(response.data);
}
export async function deleteProject(id: string): Promise<void> {
await apiClient.delete(`/api/projects/${id}`);
}
// Templates
function _mapTemplate(t: any): Template {
const mapping = t.mapping_rules || {};
return {
id: String(t.id),
name: t.name,
description: t.description,
classes: mapping.classes || [],
rules: mapping.rules || [],
createdAt: t.created_at,
updatedAt: t.updated_at,
};
}
export async function getTemplates(): Promise<Template[]> {
const response = await apiClient.get('/api/templates');
return response.data.map(_mapTemplate);
}
export async function createTemplate(payload: {
name: string;
description?: string;
color: string;
z_index: number;
classes?: { name: string; color: string; zIndex: number; category?: string }[];
rules?: any[];
}): Promise<Template> {
const response = await apiClient.post('/api/templates', payload);
return _mapTemplate(response.data);
}
export async function updateTemplate(id: string, payload: Partial<Template> & { color?: string; z_index?: number }): Promise<Template> {
const response = await apiClient.patch(`/api/templates/${id}`, payload);
return _mapTemplate(response.data);
}
export async function deleteTemplate(id: string): Promise<void> {
await apiClient.delete(`/api/templates/${id}`);
}
// Media
export async function uploadMedia(file: File, projectId?: string): Promise<{ url: string; id: string }> {
const formData = new FormData();
formData.append('file', file);
if (projectId) {
formData.append('project_id', projectId);
}
const response = await apiClient.post('/api/media/upload', formData, {
headers: {
'Content-Type': 'multipart/form-data',
},
});
const { file_url, object_name } = response.data;
return { url: file_url, id: object_name };
}
export async function getProjectFrames(projectId: string): Promise<Array<{
id: number;
project_id: number;
frame_index: number;
image_url: string;
width: number | null;
height: number | null;
timestamp_ms?: number | null;
source_frame_number?: number | null;
}>> {
const response = await apiClient.get(`/api/projects/${projectId}/frames`);
return response.data;
}
export async function uploadDicomBatch(files: File[], projectId?: string): Promise<{ project_id: number; uploaded_count: number; message: string }> {
const formData = new FormData();
files.forEach((file) => formData.append('files', file));
if (projectId) formData.append('project_id', projectId);
const response = await apiClient.post('/api/media/upload/dicom', formData, {
headers: { 'Content-Type': 'multipart/form-data' },
});
return response.data;
}
export interface ProcessingTask {
id: number;
task_type: string;
status: 'queued' | 'running' | 'success' | 'failed' | string;
progress: number;
message?: string | null;
project_id?: number | null;
celery_task_id?: string | null;
payload?: Record<string, unknown> | null;
result?: Record<string, unknown> | null;
error?: string | null;
created_at: string;
started_at?: string | null;
finished_at?: string | null;
updated_at: string;
}
export async function parseMedia(projectId: string, options: {
parseFps?: number;
maxFrames?: number;
targetWidth?: number;
} = {}): Promise<ProcessingTask> {
const response = await apiClient.post('/api/media/parse', null, {
params: {
project_id: projectId,
...(options.parseFps ? { parse_fps: options.parseFps } : {}),
...(options.maxFrames ? { max_frames: options.maxFrames } : {}),
...(options.targetWidth ? { target_width: options.targetWidth } : {}),
},
});
return response.data;
}
export async function getTask(taskId: string | number): Promise<ProcessingTask> {
const response = await apiClient.get(`/api/tasks/${taskId}`);
return response.data;
}
export async function cancelTask(taskId: string | number): Promise<ProcessingTask> {
const response = await apiClient.post(`/api/tasks/${taskId}/cancel`);
return response.data;
}
export async function retryTask(taskId: string | number): Promise<ProcessingTask> {
const response = await apiClient.post(`/api/tasks/${taskId}/retry`);
return response.data;
}
interface PredictMaskPayload {
imageId: string;
imageWidth: number;
imageHeight: number;
model?: AiModelId;
points?: { x: number; y: number; type: 'pos' | 'neg' }[];
box?: { x1: number; y1: number; x2: number; y2: number };
text?: string;
options?: {
crop_to_prompt?: boolean;
auto_filter_background?: boolean;
min_score?: number;
crop_margin?: number;
};
}
interface PredictMaskResult {
masks: Array<{
id: string;
pathData: string;
label: string;
color: string;
segmentation: number[][];
bbox: [number, number, number, number];
area: number;
confidence: number;
}>;
}
export interface AiModelStatus {
id: AiModelId;
label: string;
available: boolean;
loaded: boolean;
device: string;
supports: string[];
message: string;
package_available: boolean;
checkpoint_exists: boolean;
checkpoint_path?: string | null;
python_ok: boolean;
torch_ok: boolean;
cuda_required: boolean;
external_available?: boolean;
external_python?: string | null;
}
export interface AiRuntimeStatus {
selected_model: AiModelId;
gpu: {
available: boolean;
device: string;
name?: string | null;
torch_available: boolean;
torch_version?: string | null;
cuda_version?: string | null;
};
models: AiModelStatus[];
}
export interface SavedAnnotation {
id: number;
project_id: number;
frame_id: number | null;
template_id: number | null;
mask_data: {
polygons?: number[][][];
label?: string;
color?: string;
class?: {
id?: string;
name?: string;
color?: string;
zIndex?: number;
category?: string;
};
} | null;
points: number[][] | null;
bbox: number[] | null;
created_at: string;
updated_at: string;
}
export interface SaveAnnotationPayload {
project_id: number;
frame_id?: number;
template_id?: number;
mask_data?: {
polygons: number[][][];
label?: string;
color?: string;
class?: {
id?: string;
name?: string;
color?: string;
zIndex?: number;
category?: string;
};
};
points?: number[][];
bbox?: number[];
}
export type UpdateAnnotationPayload = Omit<SaveAnnotationPayload, 'project_id' | 'frame_id'>;
export interface PropagateMasksPayload {
project_id: number;
frame_id: number;
model?: AiModelId;
seed: {
polygons?: number[][][];
bbox?: number[];
points?: number[][];
label?: string;
color?: string;
class_metadata?: {
id?: string;
name?: string;
color?: string;
zIndex?: number;
category?: string;
};
template_id?: number;
};
direction?: 'forward' | 'backward' | 'both';
max_frames?: number;
include_source?: boolean;
save_annotations?: boolean;
}
export interface PropagateMasksResult {
model: AiModelId;
direction: string;
source_frame_id: number;
processed_frame_count: number;
created_annotation_count: number;
annotations: SavedAnnotation[];
}
export interface DashboardTask {
id: string;
task_id?: number;
project_id: number;
name: string;
progress: number;
status: string;
raw_status?: string;
error?: string | null;
frame_count: number;
updated_at: string | null;
}
export interface DashboardActivity {
id: string;
kind: 'project' | 'annotation' | 'template' | string;
time: string | null;
message: string;
project: string;
}
export interface DashboardOverview {
summary: {
project_count: number;
parsing_task_count: number;
annotation_count: number;
frame_count: number;
template_count: number;
system_load_percent: number;
};
tasks: DashboardTask[];
activity: DashboardActivity[];
}
function clamp01(value: number): number {
return Math.min(Math.max(value, 0), 1);
}
function normalizePoint(point: { x: number; y: number }, width: number, height: number): [number, number] {
return [
clamp01(point.x / Math.max(width, 1)),
clamp01(point.y / Math.max(height, 1)),
];
}
function polygonToPath(points: number[][], width: number, height: number): string {
if (points.length === 0) return '';
return points
.map(([x, y], index) => {
const px = x * width;
const py = y * height;
return `${index === 0 ? 'M' : 'L'} ${px} ${py}`;
})
.join(' ')
.concat(' Z');
}
function polygonToBbox(points: number[][], width: number, height: number): [number, number, number, number] {
const xs = points.map(([x]) => x * width);
const ys = points.map(([, y]) => y * height);
const minX = Math.min(...xs);
const minY = Math.min(...ys);
const maxX = Math.max(...xs);
const maxY = Math.max(...ys);
return [minX, minY, maxX - minX, maxY - minY];
}
function pixelSegmentationToNormalizedPolygons(
segmentation: number[][] | undefined,
width: number,
height: number,
): number[][][] {
if (!segmentation) return [];
return segmentation
.map((poly) => {
const points: number[][] = [];
for (let i = 0; i < poly.length - 1; i += 2) {
points.push([
clamp01(poly[i] / Math.max(width, 1)),
clamp01(poly[i + 1] / Math.max(height, 1)),
]);
}
return points;
})
.filter((points) => points.length > 0);
}
export function buildAnnotationPayload(
projectId: string,
mask: Mask,
frame: Frame,
templateId?: string | null,
): SaveAnnotationPayload | null {
const polygons = pixelSegmentationToNormalizedPolygons(mask.segmentation, frame.width, frame.height);
if (polygons.length === 0) return null;
const effectiveTemplateId = mask.templateId || templateId || undefined;
const classMetadata = mask.classId || mask.className || mask.classZIndex !== undefined
? {
id: mask.classId,
name: mask.className || mask.label,
color: mask.color,
zIndex: mask.classZIndex,
}
: undefined;
const payload: SaveAnnotationPayload = {
project_id: Number(projectId),
frame_id: Number(frame.id),
template_id: effectiveTemplateId ? Number(effectiveTemplateId) : undefined,
mask_data: {
polygons,
label: mask.label,
color: mask.color,
...(classMetadata ? { class: classMetadata } : {}),
},
bbox: mask.bbox
? [
clamp01(mask.bbox[0] / Math.max(frame.width, 1)),
clamp01(mask.bbox[1] / Math.max(frame.height, 1)),
clamp01(mask.bbox[2] / Math.max(frame.width, 1)),
clamp01(mask.bbox[3] / Math.max(frame.height, 1)),
]
: undefined,
};
if (mask.points) {
payload.points = mask.points.map(([x, y]) => [
clamp01(x / Math.max(frame.width, 1)),
clamp01(y / Math.max(frame.height, 1)),
]);
}
return payload;
}
export function annotationToMask(annotation: SavedAnnotation, frame: Frame): Mask | null {
const polygons = annotation.mask_data?.polygons || [];
const firstPolygon = polygons[0];
if (!firstPolygon || firstPolygon.length === 0) return null;
const bbox = polygonToBbox(firstPolygon, frame.width, frame.height);
const classMetadata = annotation.mask_data?.class;
return {
id: `annotation-${annotation.id}`,
annotationId: String(annotation.id),
frameId: String(annotation.frame_id),
templateId: annotation.template_id ? String(annotation.template_id) : undefined,
classId: classMetadata?.id,
className: classMetadata?.name,
classZIndex: classMetadata?.zIndex,
saveStatus: 'saved',
saved: true,
pathData: polygonToPath(firstPolygon, frame.width, frame.height),
label: classMetadata?.name || annotation.mask_data?.label || `Annotation ${annotation.id}`,
color: classMetadata?.color || annotation.mask_data?.color || '#06b6d4',
segmentation: polygons.map((polygon) => polygon.flatMap(([x, y]) => [x * frame.width, y * frame.height])),
points: annotation.points?.map(([x, y]) => [x * frame.width, y * frame.height]),
bbox,
area: bbox[2] * bbox[3],
};
}
export async function predictMask(payload: PredictMaskPayload): Promise<PredictMaskResult> {
let prompt_type: 'point' | 'box' | 'semantic' | 'interactive';
let prompt_data: unknown;
if (payload.box && payload.points && payload.points.length > 0) {
prompt_type = 'interactive';
prompt_data = {
box: [
clamp01(payload.box.x1 / Math.max(payload.imageWidth, 1)),
clamp01(payload.box.y1 / Math.max(payload.imageHeight, 1)),
clamp01(payload.box.x2 / Math.max(payload.imageWidth, 1)),
clamp01(payload.box.y2 / Math.max(payload.imageHeight, 1)),
],
points: payload.points.map((point) => normalizePoint(point, payload.imageWidth, payload.imageHeight)),
labels: payload.points.map((point) => (point.type === 'neg' ? 0 : 1)),
};
} else if (payload.box) {
prompt_type = 'box';
prompt_data = [
clamp01(payload.box.x1 / Math.max(payload.imageWidth, 1)),
clamp01(payload.box.y1 / Math.max(payload.imageHeight, 1)),
clamp01(payload.box.x2 / Math.max(payload.imageWidth, 1)),
clamp01(payload.box.y2 / Math.max(payload.imageHeight, 1)),
];
} else if (payload.points && payload.points.length > 0) {
prompt_type = 'point';
prompt_data = {
points: payload.points.map((point) => normalizePoint(point, payload.imageWidth, payload.imageHeight)),
labels: payload.points.map((point) => (point.type === 'neg' ? 0 : 1)),
};
} else {
prompt_type = 'semantic';
prompt_data = payload.text?.trim() || '';
}
const response = await apiClient.post('/api/ai/predict', {
image_id: Number(payload.imageId),
prompt_type,
prompt_data,
model: payload.model || 'sam2',
...(payload.options ? { options: payload.options } : {}),
});
const polygons: number[][][] = response.data.polygons || [];
const scores: number[] = response.data.scores || [];
return {
masks: polygons.map((polygon, index) => {
const bbox = polygonToBbox(polygon, payload.imageWidth, payload.imageHeight);
return {
id: `mask-${payload.imageId}-${Date.now()}-${index}`,
pathData: polygonToPath(polygon, payload.imageWidth, payload.imageHeight),
label: prompt_type === 'semantic' ? (payload.text?.trim() || 'AI Mask') : 'AI Mask',
color: '#06b6d4',
segmentation: [polygon.flatMap(([x, y]) => [x * payload.imageWidth, y * payload.imageHeight])],
bbox,
area: bbox[2] * bbox[3],
confidence: scores[index] ?? 0,
};
}),
};
}
export async function getAiModelStatus(selectedModel?: AiModelId): Promise<AiRuntimeStatus> {
const response = await apiClient.get('/api/ai/models/status', {
params: selectedModel ? { selected_model: selectedModel } : undefined,
});
return response.data;
}
export async function getProjectAnnotations(projectId: string, frameId?: string): Promise<SavedAnnotation[]> {
const response = await apiClient.get('/api/ai/annotations', {
params: {
project_id: Number(projectId),
...(frameId ? { frame_id: Number(frameId) } : {}),
},
});
return response.data;
}
export async function propagateMasks(payload: PropagateMasksPayload): Promise<PropagateMasksResult> {
const response = await apiClient.post('/api/ai/propagate', payload, {
timeout: 600000,
});
return response.data;
}
export async function saveAnnotation(payload: SaveAnnotationPayload): Promise<SavedAnnotation> {
const response = await apiClient.post('/api/ai/annotate', payload);
return response.data;
}
export async function updateAnnotation(annotationId: string, payload: UpdateAnnotationPayload): Promise<SavedAnnotation> {
const response = await apiClient.patch(`/api/ai/annotations/${annotationId}`, payload);
return response.data;
}
export async function deleteAnnotation(annotationId: string): Promise<void> {
await apiClient.delete(`/api/ai/annotations/${annotationId}`);
}
export async function importGtMask(
file: File,
projectId: string,
frameId: string,
templateId?: string | null,
): Promise<SavedAnnotation[]> {
const formData = new FormData();
formData.append('file', file);
formData.append('project_id', projectId);
formData.append('frame_id', frameId);
if (templateId) formData.append('template_id', templateId);
const response = await apiClient.post('/api/ai/import-gt-mask', formData, {
headers: { 'Content-Type': 'multipart/form-data' },
});
return response.data;
}
export async function getDashboardOverview(): Promise<DashboardOverview> {
const response = await apiClient.get('/api/dashboard/overview');
return response.data;
}
// Export
export async function exportCoco(projectId: string): Promise<Blob> {
const response = await apiClient.get(`/api/export/${projectId}/coco`, {
responseType: 'blob',
});
return response.data;
}
export async function exportMasks(projectId: string): Promise<Blob> {
const response = await apiClient.get(`/api/export/${projectId}/masks`, {
responseType: 'blob',
});
return response.data;
}
export default apiClient;