feat: AI面板图片联动+聊天图片展示+讯飞语音识别

- AI图片选择区改为仅展示编辑器中已插入的占位图
- 用户发送的图片在聊天气泡中展示并包含在导出日志中
- 接入讯飞Spark IAT流式听写WebSocket替换原生语音识别

src/pages/Login.tsx

@@ -72,7 +72,8 @@ export default function Login() {
           aiProviders: { ...DEFAULT_AI_PROVIDERS, kimi: { ...DEFAULT_AI_PROVIDERS.kimi, apiKey: getDefaultApiKey() } },
           autoInsertFrames: true,
           autoInsertDelay: 1,
-          autoInsertFrameIndices: [0, 2, 4, 6, 8, 10]
+          autoInsertFrameIndices: [0, 2, 4, 6, 8, 10],
+          xfIatConfig: { appId: 'e0fe23e3', apiKey: '7fd08be316718c2280e85af4fe126306', apiSecret: 'ZGI5MjAzZDA0YzYwNDhjMWZiNTM2NDE0' }
         };
         storage.set('systemSettings', defaultSettings);
       }

src/pages/ReportEditor.tsx

@@ -60,14 +60,18 @@ export default function ReportEditor() {
 
   // AI 撰写相关核心状态
   const [chatInput, setChatInput] = useState<string>('');
-  const [chatMessages, setChatMessages] = useState<{id: string, role: 'user'|'model', content: string}[]>([]);
+  const [chatMessages, setChatMessages] = useState<{id: string, role: 'user'|'model', content: string, images?: string[]}[]>([]);
   const [isGenerating, setIsGenerating] = useState(false);
-  const [aiSelectedFrames, setAiSelectedFrames] = useState<number[]>([]);
+
   const [aiTargetRegion, setAiTargetRegion] = useState<string>('none');
   const [aiModifyEnabled, setAiModifyEnabled] = useState(true);
   const [isListening, setIsListening] = useState(false);
   const [aiUploadedImages, setAiUploadedImages] = useState<{id: number, dataUrl: string}[]>([]);
-  const speechRecognitionRef = useRef<any>(null);
+  const [editorImages, setEditorImages] = useState<{id: string, src: string}[]>([]);
+  const [aiSelectedEditorImages, setAiSelectedEditorImages] = useState<string[]>([]);
+  const xfWsRef = useRef<WebSocket | null>(null);
+  const xfAudioContextRef = useRef<AudioContext | null>(null);
+  const xfMediaStreamRef = useRef<MediaStream | null>(null);
   const [quickPrompts, setQuickPrompts] = useState<string[]>([
     '请完善报告内容', '请对内容做如下修改：'
   ]);
@@ -85,6 +89,28 @@ export default function ReportEditor() {
     stateRef.current.chatMessages = chatMessages;
   }, [chatMessages]);
 
+  // 监听编辑器中已插入的图片，同步到 AI 面板
+  useEffect(() => {
+    if (!editorRef.current) return;
+    const updateEditorImages = () => {
+      if (!editorRef.current) return;
+      const imgs = Array.from(editorRef.current.querySelectorAll('.image-placeholder.has-image img'))
+        .map((img, idx) => ({ id: `editor-img-${idx}-${(img as HTMLImageElement).src.slice(-16)}`, src: (img as HTMLImageElement).src }))
+        .filter(img => img.src);
+      setEditorImages(prev => {
+        const same = prev.length === imgs.length && prev.every((p, i) => p.src === imgs[i]?.src);
+        if (same) return prev;
+        // 清除已不存在的选中项
+        setAiSelectedEditorImages(prevSelected => prevSelected.filter(id => imgs.some(img => img.id === id)));
+        return imgs;
+      });
+    };
+    updateEditorImages();
+    const observer = new MutationObserver(updateEditorImages);
+    observer.observe(editorRef.current, { childList: true, subtree: true, attributes: true });
+    return () => observer.disconnect();
+  }, []);
+
   useEffect(() => {
     stateRef.current.chatInput = chatInput;
   }, [chatInput]);
@@ -853,44 +879,107 @@ export default function ReportEditor() {
     return html;
   };
 
-  const toggleListening = () => {
+  async function getXfAuthUrl(apiKey: string, apiSecret: string): Promise<string> {
+    const host = 'iat-api.xfyun.cn';
+    const date = new Date().toUTCString();
+    const signatureOrigin = `host: "${host}"\ndate: "${date}"\nGET /v2/iat HTTP/1.1`;
+    const encoder = new TextEncoder();
+    const key = await crypto.subtle.importKey('raw', encoder.encode(apiSecret), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']);
+    const signature = await crypto.subtle.sign('HMAC', key, encoder.encode(signatureOrigin));
+    const signatureBase64 = btoa(String.fromCharCode(...new Uint8Array(signature)));
+    const authorizationOrigin = `api_key="${apiKey}", algorithm="hmac-sha256", headers="host date request-line", signature="${signatureBase64}"`;
+    const authorization = btoa(authorizationOrigin);
+    return `wss://iat-api.xfyun.cn/v2/iat?authorization=${encodeURIComponent(authorization)}&date=${encodeURIComponent(date)}&host=${encodeURIComponent(host)}`;
+  }
+
+  function floatTo16BitPCM(input: Float32Array): ArrayBuffer {
+    const output = new DataView(new ArrayBuffer(input.length * 2));
+    for (let i = 0; i < input.length; i++) {
+      const s = Math.max(-1, Math.min(1, input[i]));
+      output.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
+    }
+    return output.buffer;
+  }
+
+  function arrayBufferToBase64(buffer: ArrayBuffer): string {
+    const bytes = new Uint8Array(buffer);
+    let binary = '';
+    for (let i = 0; i < bytes.byteLength; i++) {
+      binary += String.fromCharCode(bytes[i]);
+    }
+    return btoa(binary);
+  }
+
+  const toggleListening = async () => {
     if (isListening) {
       setIsListening(false);
-      if (speechRecognitionRef.current) speechRecognitionRef.current.stop();
+      if (xfWsRef.current) { try { xfWsRef.current.close(); } catch {} xfWsRef.current = null; }
-    } else {
+      if (xfAudioContextRef.current) { try { xfAudioContextRef.current.close(); } catch {} xfAudioContextRef.current = null; }
-      const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
+      if (xfMediaStreamRef.current) { xfMediaStreamRef.current.getTracks().forEach(t => t.stop()); xfMediaStreamRef.current = null; }
-      if (!SpeechRecognition) {
-        alert('您的浏览器不支持原生语音识别，请使用 Chrome。');
       return;
     }
-      const recognition = new SpeechRecognition();
+    const xfConfig = storage.get<SystemSettings>('systemSettings', {} as SystemSettings).xfSpeechConfig;
-      recognition.lang = 'zh-CN';
+    if (!xfConfig?.appId || !xfConfig?.apiKey || !xfConfig?.apiSecret) {
-      recognition.continuous = true;
+      alert('请先在系统设置中配置讯飞语音 APPID/APIKey/APISecret');
-      recognition.interimResults = true;
+      return;
-      let finalTranscript = chatInput;
-      recognition.onstart = () => setIsListening(true);
-      recognition.onresult = (event: any) => {
-        let interimTranscript = '';
-        for (let i = event.resultIndex; i < event.results.length; ++i) {
-          if (event.results[i].isFinal) {
-            finalTranscript += event.results[i][0].transcript;
-          } else {
-            interimTranscript += event.results[i][0].transcript;
     }
-        }
+    try {
-        setChatInput(finalTranscript + interimTranscript);
+      const authUrl = await getXfAuthUrl(xfConfig.apiKey, xfConfig.apiSecret);
+      const ws = new WebSocket(authUrl);
+      xfWsRef.current = ws;
+      let frameStatus = 0;
+      let transcript = chatInput;
+      ws.onopen = async () => {
+        try {
+          const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+          xfMediaStreamRef.current = stream;
+          const audioContext = new AudioContext({ sampleRate: 16000 });
+          xfAudioContextRef.current = audioContext;
+          const source = audioContext.createMediaStreamSource(stream);
+          const processor = audioContext.createScriptProcessor(4096, 1, 1);
+          processor.onaudioprocess = (e) => {
+            if (ws.readyState !== WebSocket.OPEN) return;
+            const inputData = e.inputBuffer.getChannelData(0);
+            const pcmBuffer = floatTo16BitPCM(inputData);
+            const base64Audio = arrayBufferToBase64(pcmBuffer);
+            const frame: any = { data: { status: frameStatus, format: 'audio/L16;rate=16000', encoding: 'raw', audio: base64Audio } };
+            if (frameStatus === 0) { frame.common = { app_id: xfConfig.appId }; frame.business = { language: 'zh_cn', domain: 'iat', accent: 'mandarin' }; }
+            ws.send(JSON.stringify(frame));
+            frameStatus = 1;
           };
-      recognition.onerror = () => setIsListening(false);
+          source.connect(processor);
-      recognition.onend = () => setIsListening(false);
+          processor.connect(audioContext.destination);
-      speechRecognitionRef.current = recognition;
+          setIsListening(true);
-      recognition.start();
+        } catch (e: any) {
+          alert('麦克风启动失败: ' + e.message);
+          setIsListening(false);
+          ws.close();
+        }
+      };
+      ws.onmessage = (event) => {
+        try {
+          const jsonData = JSON.parse(event.data);
+          if (jsonData.data?.result?.ws) {
+            let seg = '';
+            for (const w of jsonData.data.result.ws) { if (w.cw?.[0]?.w) seg += w.cw[0].w; }
+            if (jsonData.data.result.ls) { transcript += seg; setChatInput(transcript); }
+            else { setChatInput(transcript + seg); }
+          }
+        } catch {}
+      };
+      ws.onerror = () => { alert('讯飞语音连接失败'); setIsListening(false); };
+      ws.onclose = () => { setIsListening(false); };
+    } catch (e: any) {
+      alert('讯飞语音初始化失败: ' + e.message);
     }
   };
 
   const handleAIGenerate = async (text: string) => {
     if (!text.trim()) return;
     const userMsgId = Date.now().toString();
-    setChatMessages(prev => [...prev, { id: userMsgId, role: 'user', content: text }]);
+    const selectedEditorImageUrls = editorImages.filter(img => aiSelectedEditorImages.includes(img.id)).map(img => img.src);
+    const allImages = [...selectedEditorImageUrls, ...aiUploadedImages.map(i => i.dataUrl)];
+    setChatMessages(prev => [...prev, { id: userMsgId, role: 'user', content: text, images: allImages.length > 0 ? allImages : undefined }]);
     setChatInput('');
     setIsGenerating(true);
     try {
@@ -933,8 +1022,8 @@ export default function ReportEditor() {
       const currentHtml = targetRegionEl ? targetRegionEl.innerHTML.replace(/&#8203;/g, '').replace(/>(\s+)</g, '><').trim() : '';
       const globalContextText = editorRef.current?.innerText || '';
       let messageContent: any;
-      const selectedFrameUrls = aiSelectedFrames.map(id => capturedFrames.find(f => f.id === id)?.dataUrl).filter(Boolean);
+      const selectedEditorImageUrls = editorImages.filter(img => aiSelectedEditorImages.includes(img.id)).map(img => img.src);
-      const allImages = [...selectedFrameUrls, ...aiUploadedImages.map(i => i.dataUrl)];
+      const allImages = [...selectedEditorImageUrls, ...aiUploadedImages.map(i => i.dataUrl)];
       let promptText = `【全局手术报告参考内容】:\n${globalContextText}\n\n`;
       if (aiModifyEnabled && targetRegionEl) {
         promptText += `【你需要进行修改的目标区域 HTML 源码】:\n${currentHtml || '（当前区域为空）'}\n\n`;
@@ -1031,7 +1120,7 @@ export default function ReportEditor() {
         }
       }
       setAiUploadedImages([]);
-      setAiSelectedFrames([]);
+      setAiSelectedEditorImages([]);
     } catch (error: any) {
       console.error(error);
       setChatMessages(prev => [...prev, { id: Date.now().toString(), role: 'model', content: `【系统错误】: ${error.message}` }]);
@@ -1243,7 +1332,7 @@ export default function ReportEditor() {
         setChatMessages([]);
         setChatInput('');
         setAiUploadedImages([]);
-        setAiSelectedFrames([]);
+        setAiSelectedEditorImages([]);
         prevVideoCountRef.current = 0;
         stateRef.current = {
           ...stateRef.current,
@@ -2331,7 +2420,14 @@ export default function ReportEditor() {
                       chatMessages.map(msg => (
                         <div key={msg.id} className={`flex ${msg.role === 'user' ? 'justify-end' : 'justify-start'}`}>
                           <div className={`rounded-2xl px-4 py-2.5 max-w-[85%] text-sm ${msg.role === 'user' ? 'bg-blue-600 text-white rounded-tr-none shadow-md' : 'bg-white border border-slate-200 text-slate-700 rounded-tl-none shadow-sm'}`}>
-                            {msg.content}
+                            <div>{msg.content}</div>
+                            {msg.images && msg.images.length > 0 && (
+                              <div className="flex gap-1.5 mt-2 flex-wrap">
+                                {msg.images.map((src, idx) => (
+                                  <img key={idx} src={src} className="w-10 h-10 object-cover rounded border border-white/30" />
+                                ))}
+                              </div>
+                            )}
                           </div>
                         </div>
                       ))
@@ -2381,15 +2477,15 @@ export default function ReportEditor() {
                       </label>
                     </div>
 
-                    {/* 视觉参考上下文 */}
+                    {/* 视觉参考上下文 - 编辑器中已插入的图片 */}
-                    {capturedFrames.length > 0 && (
+                    {editorImages.length > 0 && (
                       <div className="flex gap-2 overflow-x-auto pb-1 custom-scrollbar">
-                        {capturedFrames.map(frame => {
+                        {editorImages.map(img => {
-                          const isSelected = aiSelectedFrames.includes(frame.id);
+                          const isSelected = aiSelectedEditorImages.includes(img.id);
                           return (
-                            <div key={frame.id} onClick={() => setAiSelectedFrames(prev => isSelected ? prev.filter(id => id !== frame.id) : [...prev, frame.id])}
+                            <div key={img.id} onClick={() => setAiSelectedEditorImages(prev => isSelected ? prev.filter(id => id !== img.id) : [...prev, img.id])}
                               className={`relative shrink-0 w-12 aspect-video rounded overflow-hidden border-2 cursor-pointer transition-all ${isSelected ? 'border-blue-600' : 'border-transparent opacity-50'}`}>
-                              <img src={frame.dataUrl} className="w-full h-full object-cover" />
+                              <img src={img.src} className="w-full h-full object-cover" />
                               {isSelected && <div className="absolute top-0.5 right-0.5 bg-blue-600 rounded-full p-0.5"><Check size={8} className="text-white" /></div>}
                             </div>
                           );
@@ -2435,7 +2531,7 @@ export default function ReportEditor() {
                             modifyEnabled: aiModifyEnabled,
                             chatInput,
                             uploadedImagesCount: aiUploadedImages.length,
-                            selectedFramesCount: aiSelectedFrames.length
+                            selectedFramesCount: aiSelectedEditorImages.length
                           }
                         };
                         const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });

src/pages/SystemSettings.tsx

@@ -14,7 +14,8 @@ export default function SystemSettings() {
     defaultTemplate: '',
     frameMode: 'keep',
     activeAiProvider: 'kimi',
-    aiProviders: { ...DEFAULT_AI_PROVIDERS }
+    aiProviders: { ...DEFAULT_AI_PROVIDERS },
+    xfIatConfig: { appId: 'e0fe23e3', apiKey: '7fd08be316718c2280e85af4fe126306', apiSecret: 'ZGI5MjAzZDA0YzYwNDhjMWZiNTM2NDE0' }
   });
   const [templates, setTemplates] = useState<Template[]>([]);
   const [isSaved, setIsSaved] = useState(false);
@@ -68,6 +69,7 @@ export default function SystemSettings() {
       if (!savedSettings.frameMode) savedSettings.frameMode = 'keep';
       if (typeof savedSettings.autoInsertFrames !== 'boolean') savedSettings.autoInsertFrames = false;
       if (typeof savedSettings.autoInsertDelay !== 'number') savedSettings.autoInsertDelay = 0;
+      if (!savedSettings.xfSpeechConfig) savedSettings.xfSpeechConfig = { appId: 'e0fe23e3', apiKey: '7fd08be316718c2280e85af4fe126306', apiSecret: 'ZGI5MjAzZDA0YzYwNDhjMWZiNTM2NDE0' };
       setSettings(savedSettings);
     } else if (savedTemplates.length > 0) {
       setSettings(prev => ({ ...prev, defaultTemplate: savedTemplates[0].id, frameMode: prev.frameMode || 'keep', autoInsertFrames: typeof prev.autoInsertFrames === 'boolean' ? prev.autoInsertFrames : false, autoInsertDelay: typeof prev.autoInsertDelay === 'number' ? prev.autoInsertDelay : 0 }));
@@ -150,7 +152,8 @@ export default function SystemSettings() {
         aiProviders: { ...DEFAULT_AI_PROVIDERS, kimi: { ...DEFAULT_AI_PROVIDERS.kimi, apiKey: getDefaultApiKey() } },
         autoInsertFrames: true,
         autoInsertDelay: 1,
-        autoInsertFrameIndices: [0, 2, 4, 6, 8, 10]
+        autoInsertFrameIndices: [0, 2, 4, 6, 8, 10],
+        xfSpeechConfig: { appId: 'e0fe23e3', apiKey: '7fd08be316718c2280e85af4fe126306', apiSecret: 'ZGI5MjAzZDA0YzYwNDhjMWZiNTM2NDE0' }
       };
       setSettings(defaultSettings);
       storage.set('systemSettings', defaultSettings);
@@ -421,6 +424,58 @@ export default function SystemSettings() {
           </div>
           )}
 
+          {currentUser.role === 'super' && (
+          <div className="card-minimal">
+            <div className="flex items-center justify-between mb-6">
+              <h3 className="text-lg font-bold text-text-main flex items-center gap-2">
+                <svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round" className="text-accent"><path d="M12 2a3 3 0 0 0-3 3v7a3 3 0 0 0 6 0V5a3 3 0 0 0-3-3Z"/><path d="M19 10v2a7 7 0 0 1-14 0v-2"/><line x1="12" x2="12" y1="19" y2="22"/></svg>
+                讯飞语音配置
+              </h3>
+            </div>
+            <div className="space-y-6">
+              <div className="space-y-1.5">
+                <label className="block text-xs font-bold text-text-main uppercase tracking-wider">APPID</label>
+                <input 
+                  type="password" 
+                  value={settings.xfIatConfig?.appId || ''}
+                  onChange={(e) => {
+                    const next = { ...(settings.xfIatConfig || { appId: '', apiKey: '', apiSecret: '' }), appId: e.target.value };
+                    setSettings({ ...settings, xfIatConfig: next });
+                  }}
+                  placeholder="e0fe23e3"
+                  className="input-minimal"
+                />
+              </div>
+              <div className="space-y-1.5">
+                <label className="block text-xs font-bold text-text-main uppercase tracking-wider">APIKey</label>
+                <input 
+                  type="password" 
+                  value={settings.xfIatConfig?.apiKey || ''}
+                  onChange={(e) => {
+                    const next = { ...(settings.xfIatConfig || { appId: '', apiKey: '', apiSecret: '' }), apiKey: e.target.value };
+                    setSettings({ ...settings, xfIatConfig: next });
+                  }}
+                  placeholder="7fd08be316718c2280e85af4fe126306"
+                  className="input-minimal"
+                />
+              </div>
+              <div className="space-y-1.5">
+                <label className="block text-xs font-bold text-text-main uppercase tracking-wider">APISecret</label>
+                <input 
+                  type="password" 
+                  value={settings.xfIatConfig?.apiSecret || ''}
+                  onChange={(e) => {
+                    const next = { ...(settings.xfIatConfig || { appId: '', apiKey: '', apiSecret: '' }), apiSecret: e.target.value };
+                    setSettings({ ...settings, xfIatConfig: next });
+                  }}
+                  placeholder="ZGI5MjAzZDA0YzYwNDhjMWZiNTM2NDE0"
+                  className="input-minimal"
+                />
+              </div>
+            </div>
+          </div>
+          )}
+
           <div className="card-minimal">
             <h3 className="text-lg font-bold text-text-main mb-6 flex items-center gap-2">
               <Layout size={20} className="text-accent" />

src/types.ts

@@ -86,6 +86,7 @@ export interface SystemSettings {
   autoInsertDelay?: number;
   activeAiProvider: string;
   aiProviders: Record<string, AiProviderConfig>;
+  xfSpeechConfig?: { appId: string; apiKey: string; apiSecret: string };
 }
 
 export const DEFAULT_AI_PROVIDERS: Record<string, AiProviderConfig> = {