2026-04-19-02-26-05 集成AI撰写功能:Kimi-2.5多模态API、AI可编辑区域、Diff确认弹窗、语音与图片输入、快捷指令

This commit is contained in:
2026-04-19 02:36:20 +08:00
parent 96b295f919
commit 221daf61a5
10 changed files with 1468 additions and 9 deletions

View File

@@ -5,7 +5,8 @@ import Sidebar from '../components/Sidebar';
import {
Check, Printer, Undo, Redo, Bold, Italic, Underline,
AlignLeft, AlignCenter, AlignRight, Table, Image as ImageIcon,
Video, Play, Pause, Plus, X, ChevronLeft, Download
Video, Play, Pause, Plus, X, ChevronLeft, Download,
Bot, Mic, MicOff, ImagePlus, Sparkles, Send
} from 'lucide-react';
import { User, Report, Template, CapturedFrame, SystemSettings, FormField, DEFAULT_FORM_FIELDS } from '../types';
import { defaultReportContent } from '../utils/defaultContent';
@@ -53,9 +54,25 @@ export default function ReportEditor() {
const [pendingTemplateId, setPendingTemplateId] = useState<string | null>(null);
const prevVideoCountRef = useRef(0);
const [activeTab, setActiveTab] = useState<'info' | 'video'>('info');
const [activeTab, setActiveTab] = useState<'info' | 'video' | 'ai'>('info');
const [activeFieldKey, setActiveFieldKey] = useState<string | null>(null);
// AI 撰写相关核心状态
const [chatInput, setChatInput] = useState<string>('');
const [chatMessages, setChatMessages] = useState<{id: string, role: 'user'|'model', content: string}[]>([]);
const [isGenerating, setIsGenerating] = useState(false);
const [aiSelectedFrames, setAiSelectedFrames] = useState<number[]>([]);
const [aiTargetRegion, setAiTargetRegion] = useState<string>('none');
const [aiModifyEnabled, setAiModifyEnabled] = useState(true);
const [isListening, setIsListening] = useState(false);
const [aiUploadedImages, setAiUploadedImages] = useState<{id: number, dataUrl: string}[]>([]);
const speechRecognitionRef = useRef<any>(null);
const [quickPrompts, setQuickPrompts] = useState<string[]>([
'请详细描述手术步骤', '提取术中关键病灶信息', '生成简短的术后总结', '根据截图描述游离过程'
]);
const [isEditingPrompts, setIsEditingPrompts] = useState(false);
const [diffModal, setDiffModal] = useState<{isOpen: boolean, originalHtml: string, newHtml: string, targetId: string} | null>(null);
useEffect(() => {
if (!editorRef.current) return;
const allFields = editorRef.current.querySelectorAll('.field-value');
@@ -604,6 +621,20 @@ export default function ReportEditor() {
setPlaceholderModal({ isOpen: true, width: '200', height: '200', mode: 'frame' });
};
const insertAiRegion = () => {
const name = window.prompt('请输入 AI 可编辑区域的名称(如:手术步骤、病灶描述):');
if (!name || !name.trim()) return;
if (editorRef.current?.querySelector(`[data-ai-id="${name}"]`)) {
window.alert('该区域名称已存在,请使用其他名称以保证 AI 定位准确。');
return;
}
editorRef.current?.focus();
const html = `<div class="ai-region" data-ai-id="${name}" data-ai-title="${name}" style="border: 1px dashed #3b82f6; padding: 16px 12px 12px; margin: 8px 0; position: relative; min-height: 60px; background: #f8fafc; border-radius: 6px;"><div contenteditable="false" style="position: absolute; top: -10px; right: 10px; background: #3b82f6; color: white; font-size: 10px; padding: 2px 8px; border-radius: 12px; z-index: 10; user-select: none; box-shadow: 0 2px 4px rgba(0,0,0,0.1);">${name}-AI可编辑区域</div><div class="ai-content" style="min-height: 20px;">&#8203;</div></div><p><br></p>`;
document.execCommand('insertHTML', false, html);
if (editorRef.current) contentRef.current = editorRef.current.innerHTML;
saveDraftToStorage();
};
const handleVideoUpload = (e: React.ChangeEvent<HTMLInputElement>) => {
const files = Array.from(e.target.files || []) as File[];
const newVideos = files.map(file => ({
@@ -767,6 +798,147 @@ export default function ReportEditor() {
return `${mins.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
};
const checkAiRegions = () => {
if (!editorRef.current) return [];
return Array.from(editorRef.current.querySelectorAll('.ai-region')).map((el) => {
const id = (el as HTMLElement).getAttribute('data-ai-id') || '';
const title = (el as HTMLElement).getAttribute('data-ai-title') || id;
return { id, title };
}).filter(r => r.id);
};
const toggleListening = () => {
if (isListening) {
setIsListening(false);
if (speechRecognitionRef.current) speechRecognitionRef.current.stop();
} else {
const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
if (!SpeechRecognition) {
alert('您的浏览器不支持原生语音识别,请使用 Chrome。');
return;
}
const recognition = new SpeechRecognition();
recognition.lang = 'zh-CN';
recognition.continuous = true;
recognition.interimResults = true;
let finalTranscript = chatInput;
recognition.onstart = () => setIsListening(true);
recognition.onresult = (event: any) => {
let interimTranscript = '';
for (let i = event.resultIndex; i < event.results.length; ++i) {
if (event.results[i].isFinal) {
finalTranscript += event.results[i][0].transcript;
} else {
interimTranscript += event.results[i][0].transcript;
}
}
setChatInput(finalTranscript + interimTranscript);
};
recognition.onerror = () => setIsListening(false);
recognition.onend = () => setIsListening(false);
speechRecognitionRef.current = recognition;
recognition.start();
}
};
const handleAIGenerate = async (text: string) => {
if (!text.trim()) return;
const userMsgId = Date.now().toString();
setChatMessages(prev => [...prev, { id: userMsgId, role: 'user', content: text }]);
setChatInput('');
setIsGenerating(true);
try {
const settings = storage.get<SystemSettings>('systemSettings', {} as SystemSettings);
const apiKey = settings.kimiApiKey || '';
const apiEndpoint = settings.kimiApiEndpoint || 'https://api.moonshot.cn/v1';
if (!apiKey) {
setChatMessages(prev => [...prev, { id: Date.now().toString(), role: 'model', content: '【系统提示】尚未配置 Kimi API Key请前往系统设置填写。' }]);
setIsGenerating(false);
return;
}
const targetRegionEl = editorRef.current?.querySelector(`.ai-region[data-ai-id="${aiTargetRegion}"] .ai-content`) as HTMLElement | null;
const currentHtml = targetRegionEl ? targetRegionEl.innerHTML : '';
const messageContent: any[] = [];
const selectedFrameUrls = aiSelectedFrames.map(id => capturedFrames.find(f => f.id === id)?.dataUrl).filter(Boolean);
const allImages = [...selectedFrameUrls, ...aiUploadedImages.map(i => i.dataUrl)];
allImages.forEach(url => {
messageContent.push({ type: 'image_url', image_url: { url } });
});
let promptText = `【医生指令】: ${text}`;
if (aiModifyEnabled && targetRegionEl) {
promptText = `【当前区域 HTML 源码】:\n${currentHtml}\n\n${promptText}`;
}
messageContent.push({ type: 'text', text: promptText });
const systemPrompt = aiModifyEnabled && targetRegionEl
? '你是一名专业的外科医生助理。你需要根据用户的指令及可能提供的截图,修改给定的 HTML 源码。\n重要指令你必须严格返回合法的 JSON 对象,绝对不要包含任何 Markdown 标记(如 ```json。\nJSON 格式如下:\n{ "reply": "简短的回复话术", "updatedHtml": "修改后的完整内部 HTML 代码" }'
: '你是一名专业的外科医生助理。请根据用户的指令和截图进行分析解答。\n重要指令你必须严格返回合法的 JSON 对象,绝对不要包含任何 Markdown 标记。\nJSON 格式如下:\n{ "reply": "你的分析和回答" }';
const response = await fetch(`${apiEndpoint}/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${apiKey}`
},
body: JSON.stringify({
model: 'kimi-k2-5',
messages: [
{ role: 'system', content: systemPrompt },
{ role: 'user', content: messageContent }
],
temperature: 0.3
})
});
if (!response.ok) throw new Error(`API 请求失败: ${response.status}`);
const data = await response.json();
const responseText = data.choices[0].message.content.trim();
const cleanedText = responseText.replace(/```json\n?|```/g, '');
let responseJson: any = {};
try {
responseJson = JSON.parse(cleanedText);
} catch {
const jsonMatch = cleanedText.match(/\{[\s\S]*\}/);
if (jsonMatch) responseJson = JSON.parse(jsonMatch[0]);
else throw new Error('AI 返回格式异常,无法解析 JSON');
}
if (responseJson.reply) {
setChatMessages(prev => [...prev, { id: Date.now().toString(), role: 'model', content: responseJson.reply }]);
}
if (responseJson.updatedHtml && aiModifyEnabled && targetRegionEl) {
setDiffModal({
isOpen: true,
originalHtml: currentHtml,
newHtml: responseJson.updatedHtml,
targetId: aiTargetRegion
});
}
setAiUploadedImages([]);
setAiSelectedFrames([]);
} catch (error: any) {
console.error(error);
setChatMessages(prev => [...prev, { id: Date.now().toString(), role: 'model', content: `【系统错误】: ${error.message}` }]);
} finally {
setIsGenerating(false);
}
};
const confirmAiInjection = (newHtml: string, regionId: string) => {
if (!editorRef.current) return;
const targetContent = editorRef.current.querySelector(`.ai-region[data-ai-id="${regionId}"] .ai-content`) as HTMLElement;
if (targetContent) {
targetContent.innerHTML = newHtml;
targetContent.style.transition = 'background-color 0.3s ease';
targetContent.style.backgroundColor = '#bfdbfe';
setTimeout(() => {
targetContent.style.backgroundColor = '#eff6ff';
setTimeout(() => {
targetContent.style.backgroundColor = 'transparent';
}, 800);
}, 400);
contentRef.current = editorRef.current.innerHTML;
saveDraftToStorage();
}
setDiffModal(null);
};
const handleDragStart = (e: React.DragEvent, frame: CapturedFrame) => {
e.dataTransfer.setData('frameId', frame.id.toString());
};
@@ -1482,6 +1654,7 @@ export default function ReportEditor() {
<div className="flex gap-1">
<button onClick={insertTable} className="w-9 h-9 flex items-center justify-center rounded-lg hover:bg-white text-text-muted hover:text-text-main transition-colors" title="表格"><Table size={16} /></button>
<button onClick={insertImage} className="w-9 h-9 flex items-center justify-center rounded-lg hover:bg-white text-text-muted hover:text-text-main transition-colors" title="插入图片占位符"><ImageIcon size={16} /></button>
<button onMouseDown={(e) => e.preventDefault()} onClick={insertAiRegion} className="w-9 h-9 flex items-center justify-center rounded-lg hover:bg-blue-50 text-blue-500 transition-colors" title="插入AI可编辑区域"><Bot size={16} /></button>
</div>
</div>
@@ -1509,7 +1682,7 @@ export default function ReportEditor() {
{/* Right Sidebar */}
<aside className="w-[380px] bg-sidebar-bg flex flex-col shrink-0 overflow-hidden">
<div className="flex border-b border-border">
{(['info', 'video'] as const).map(tab => (
{(['info', 'video', 'ai'] as const).map(tab => (
<button
key={tab}
onClick={() => { setActiveTab(tab); stateRef.current = { ...stateRef.current, activeTab: tab }; saveDraftToStorage(); }}
@@ -1517,7 +1690,7 @@ export default function ReportEditor() {
activeTab === tab ? 'text-accent border-accent' : 'text-text-muted border-transparent hover:text-text-main'
}`}
>
{tab === 'info' ? '基本信息' : '视频分析'}
{tab === 'info' ? '基本信息' : tab === 'video' ? '视频分析' : 'AI撰写'}
</button>
))}
</div>
@@ -2009,6 +2182,152 @@ export default function ReportEditor() {
)}
</div>
)}
{activeTab === 'ai' && (
<div className="flex flex-col h-full bg-[#f8fafc] overflow-hidden">
{/* 聊天气泡记录区 */}
<div className="flex-1 overflow-y-auto p-4 space-y-4 custom-scrollbar">
{chatMessages.length === 0 ? (
<div className="text-center flex flex-col items-center justify-center h-full text-slate-400 space-y-3">
<Bot size={48} className="text-slate-300 opacity-50" />
<p className="text-xs"> SurClaw </p>
</div>
) : (
chatMessages.map(msg => (
<div key={msg.id} className={`flex ${msg.role === 'user' ? 'justify-end' : 'justify-start'}`}>
<div className={`rounded-2xl px-4 py-2.5 max-w-[85%] text-sm ${msg.role === 'user' ? 'bg-blue-600 text-white rounded-tr-none shadow-md' : 'bg-white border border-slate-200 text-slate-700 rounded-tl-none shadow-sm'}`}>
{msg.content}
</div>
</div>
))
)}
{isGenerating && (
<div className="flex justify-start">
<div className="bg-white border border-slate-200 rounded-2xl rounded-tl-none px-4 py-3 shadow-sm flex gap-1.5 items-center">
<div className="w-2 h-2 bg-blue-500 rounded-full animate-bounce" />
<div className="w-2 h-2 bg-blue-500 rounded-full animate-bounce" style={{ animationDelay: '0.15s' }} />
<div className="w-2 h-2 bg-blue-500 rounded-full animate-bounce" style={{ animationDelay: '0.3s' }} />
</div>
</div>
)}
</div>
{/* 控制台与输入区 */}
<div className="bg-white border-t border-slate-200 p-4 space-y-3 shrink-0 shadow-[0_-4px_6px_-1px_rgba(0,0,0,0.05)] z-10">
{/* 区域锚定与沙盒控制 */}
<div className="flex items-center justify-between bg-slate-50 p-2 rounded-lg border border-slate-200">
<div className="flex items-center gap-2 flex-1">
<select
value={aiTargetRegion}
onChange={(e) => setAiTargetRegion(e.target.value)}
disabled={!aiModifyEnabled}
className="flex-1 w-0 px-2 py-1 border-none text-xs bg-transparent focus:ring-0 font-bold text-slate-700 disabled:opacity-50"
>
{checkAiRegions().length > 0 ? (
checkAiRegions().map((r: any) => <option key={r.id} value={r.id}>🎯 {r.title}</option>)
) : (
<option value="none"> AI </option>
)}
</select>
</div>
<div className="flex items-center gap-1.5 shrink-0 pl-2 border-l border-slate-300">
<input
type="checkbox" id="aiModifyEnabled"
checked={aiModifyEnabled}
onChange={(e) => setAiModifyEnabled(e.target.checked)}
className="w-3.5 h-3.5 text-blue-600 rounded border-slate-300 focus:ring-blue-500 cursor-pointer"
/>
<label htmlFor="aiModifyEnabled" className="text-[11px] text-slate-600 cursor-pointer font-bold">
</label>
</div>
</div>
{/* 视觉参考上下文 */}
{capturedFrames.length > 0 && (
<div className="flex gap-2 overflow-x-auto pb-1 custom-scrollbar">
{capturedFrames.map(frame => {
const isSelected = aiSelectedFrames.includes(frame.id);
return (
<div key={frame.id} onClick={() => setAiSelectedFrames(prev => isSelected ? prev.filter(id => id !== frame.id) : [...prev, frame.id])}
className={`relative shrink-0 w-12 aspect-video rounded overflow-hidden border-2 cursor-pointer transition-all ${isSelected ? 'border-blue-600' : 'border-transparent opacity-50'}`}>
<img src={frame.dataUrl} className="w-full h-full object-cover" />
{isSelected && <div className="absolute top-0.5 right-0.5 bg-blue-600 rounded-full p-0.5"><Check size={8} className="text-white" /></div>}
</div>
);
})}
</div>
)}
{/* 自定义快捷指令胶囊 */}
<div className="flex flex-wrap gap-1.5 max-h-16 overflow-y-auto">
{quickPrompts.map((p, i) => (
<div key={i} className="group relative">
<button onClick={() => setChatInput(p)} className="px-3 py-1 bg-[#f1f5f9] hover:bg-blue-50 hover:text-blue-600 text-slate-600 text-[11px] rounded-full transition-colors whitespace-nowrap">
{p}
</button>
{isEditingPrompts && (
<button onClick={() => setQuickPrompts(prev => prev.filter((_, idx) => idx !== i))} className="absolute -top-1 -right-1 bg-red-500 text-white rounded-full p-0.5 scale-75 shadow-sm">
<X size={10} />
</button>
)}
</div>
))}
<button onClick={() => {
if (isEditingPrompts) {
const newP = prompt('新增快捷指令:');
if (newP) setQuickPrompts([...quickPrompts, newP]);
} else {
setIsEditingPrompts(true);
}
}} className="px-2 py-1 bg-slate-100 text-slate-400 text-[11px] rounded-full hover:bg-slate-200">
{isEditingPrompts ? '+ 添加' : '⚙️'}
</button>
{isEditingPrompts && <button onClick={() => setIsEditingPrompts(false)} className="px-2 py-1 bg-blue-100 text-blue-600 text-[11px] rounded-full"></button>}
</div>
{/* 沉浸式输入框 */}
<div className="relative border border-slate-300 rounded-xl bg-white shadow-inner focus-within:ring-2 focus-within:ring-blue-500 focus-within:border-transparent transition-all">
{aiUploadedImages.length > 0 && (
<div className="flex gap-2 p-2 border-b border-slate-100 bg-slate-50 rounded-t-xl overflow-x-auto">
{aiUploadedImages.map(img => (
<div key={img.id} className="relative w-10 h-10 rounded overflow-hidden shadow-sm shrink-0">
<img src={img.dataUrl} className="w-full h-full object-cover" />
<button onClick={() => setAiUploadedImages(prev => prev.filter(i => i.id !== img.id))} className="absolute top-0 right-0 bg-red-500/80 text-white rounded-bl-md">
<X size={10} />
</button>
</div>
))}
</div>
)}
<textarea
value={chatInput}
onChange={(e) => setChatInput(e.target.value)}
onKeyDown={(e) => { if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); if (!isGenerating) handleAIGenerate(chatInput); } }}
placeholder={isListening ? '正在将语音转为文字...' : '输入需求(按 Enter 发送)...'}
className="w-full min-h-[80px] p-3 pr-12 text-sm bg-transparent outline-none resize-none custom-scrollbar"
/>
<div className="absolute bottom-2 right-2 flex items-center gap-1.5">
<label className="p-1.5 text-slate-400 hover:text-blue-600 bg-slate-50 hover:bg-blue-50 rounded-lg cursor-pointer transition-colors" title="上传外部图像">
<input type="file" accept="image/*" multiple className="hidden" onChange={(e) => {
const files = e.target.files;
if (!files) return;
Array.from(files).forEach((file: File) => {
const reader = new FileReader();
reader.onload = (ev) => { if (ev.target?.result) setAiUploadedImages(prev => [...prev, { id: Date.now(), dataUrl: ev.target!.result as string }]); };
reader.readAsDataURL(file);
});
}} />
<ImagePlus size={16} />
</label>
<button onClick={toggleListening} className={`p-1.5 rounded-lg transition-colors ${isListening ? 'text-red-500 bg-red-50 animate-pulse' : 'text-slate-400 hover:text-blue-600 bg-slate-50 hover:bg-blue-50'}`}>
{isListening ? <Mic size={16} /> : <MicOff size={16} />}
</button>
</div>
</div>
</div>
</div>
)}
</div>
</aside>
</div>
@@ -2219,6 +2538,50 @@ export default function ReportEditor() {
</div>
</div>
)}
{/* AI 修改二次确认 Diff 弹窗 */}
{diffModal && diffModal.isOpen && (
<div className="fixed inset-0 bg-slate-900/40 backdrop-blur-sm z-[100] flex items-center justify-center p-4">
<div className="bg-white rounded-2xl w-full max-w-[800px] shadow-2xl flex flex-col max-h-[85vh] overflow-hidden">
<div className="px-6 py-4 border-b border-slate-100 flex items-center justify-between bg-slate-50">
<div>
<h3 className="text-lg font-bold text-slate-800 flex items-center gap-2">
<Sparkles size={18} className="text-blue-600" />
AI
</h3>
<p className="text-xs text-slate-500 mt-1"> AI </p>
</div>
<button onClick={() => setDiffModal(null)} className="text-slate-400 hover:text-slate-600"><X size={20}/></button>
</div>
<div className="flex-1 overflow-hidden flex gap-4 p-6 bg-slate-100">
<div className="flex-1 flex flex-col bg-white border border-red-200 rounded-xl overflow-hidden shadow-sm">
<div className="bg-red-50 px-3 py-2 text-xs font-bold text-red-600 border-b border-red-100 uppercase tracking-wider"></div>
<div className="p-4 flex-1 overflow-y-auto opacity-70 cursor-not-allowed custom-scrollbar"
dangerouslySetInnerHTML={{ __html: diffModal.originalHtml }}></div>
</div>
<div className="flex-1 flex flex-col bg-white border border-green-400 rounded-xl overflow-hidden shadow-md relative">
<div className="bg-green-50 px-3 py-2 text-xs font-bold text-green-700 border-b border-green-200 uppercase tracking-wider flex justify-between">
<span>AI ()</span>
<span className="text-[10px] bg-green-200 px-1.5 py-0.5 rounded text-green-800"></span>
</div>
<div
className="p-4 flex-1 overflow-y-auto outline-none custom-scrollbar"
contentEditable
suppressContentEditableWarning
onBlur={(e) => setDiffModal(prev => prev ? { ...prev, newHtml: e.target.innerHTML } : null)}
dangerouslySetInnerHTML={{ __html: diffModal.newHtml }}
></div>
</div>
</div>
<div className="px-6 py-4 border-t border-slate-100 flex justify-end gap-3 bg-white">
<button onClick={() => setDiffModal(null)} className="px-6 py-2 rounded-lg text-slate-600 font-medium hover:bg-slate-100"></button>
<button onClick={() => confirmAiInjection(diffModal.newHtml, diffModal.targetId)} className="px-6 py-2 rounded-lg bg-blue-600 text-white font-medium hover:bg-blue-700 shadow-sm flex items-center gap-2">
<Check size={16} />
</button>
</div>
</div>
</div>
)}
</div>
);
}