From 2dbdbe02b20bc4e0d08d5e48852c150860b74d5a Mon Sep 17 00:00:00 2001 From: admin <572701190@qq.com> Date: Mon, 20 Apr 2026 00:36:55 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20AI=E9=9D=A2=E6=9D=BF=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E8=81=94=E5=8A=A8+=E8=81=8A=E5=A4=A9=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E5=B1=95=E7=A4=BA+=E8=AE=AF=E9=A3=9E=E8=AF=AD=E9=9F=B3?= =?UTF-8?q?=E8=AF=86=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - AI图片选择区改为仅展示编辑器中已插入的占位图 - 用户发送的图片在聊天气泡中展示并包含在导出日志中 - 接入讯飞Spark IAT流式听写WebSocket替换原生语音识别 --- src/pages/Login.tsx | 3 +- src/pages/ReportEditor.tsx | 182 ++++++++++++++++++++++++++--------- src/pages/SystemSettings.tsx | 59 +++++++++++- src/types.ts | 1 + 4 files changed, 199 insertions(+), 46 deletions(-) diff --git a/src/pages/Login.tsx b/src/pages/Login.tsx index d8f04e3..f729417 100644 --- a/src/pages/Login.tsx +++ b/src/pages/Login.tsx @@ -72,7 +72,8 @@ export default function Login() { aiProviders: { ...DEFAULT_AI_PROVIDERS, kimi: { ...DEFAULT_AI_PROVIDERS.kimi, apiKey: getDefaultApiKey() } }, autoInsertFrames: true, autoInsertDelay: 1, - autoInsertFrameIndices: [0, 2, 4, 6, 8, 10] + autoInsertFrameIndices: [0, 2, 4, 6, 8, 10], + xfIatConfig: { appId: 'e0fe23e3', apiKey: '7fd08be316718c2280e85af4fe126306', apiSecret: 'ZGI5MjAzZDA0YzYwNDhjMWZiNTM2NDE0' } }; storage.set('systemSettings', defaultSettings); } diff --git a/src/pages/ReportEditor.tsx b/src/pages/ReportEditor.tsx index 76a1706..bbf45bc 100644 --- a/src/pages/ReportEditor.tsx +++ b/src/pages/ReportEditor.tsx @@ -60,14 +60,18 @@ export default function ReportEditor() { // AI 撰写相关核心状态 const [chatInput, setChatInput] = useState(''); - const [chatMessages, setChatMessages] = useState<{id: string, role: 'user'|'model', content: string}[]>([]); + const [chatMessages, setChatMessages] = useState<{id: string, role: 'user'|'model', content: string, images?: string[]}[]>([]); const [isGenerating, setIsGenerating] = useState(false); - const [aiSelectedFrames, setAiSelectedFrames] = useState([]); + const [aiTargetRegion, setAiTargetRegion] = useState('none'); const [aiModifyEnabled, setAiModifyEnabled] = useState(true); const [isListening, setIsListening] = useState(false); const [aiUploadedImages, setAiUploadedImages] = useState<{id: number, dataUrl: string}[]>([]); - const speechRecognitionRef = useRef(null); + const [editorImages, setEditorImages] = useState<{id: string, src: string}[]>([]); + const [aiSelectedEditorImages, setAiSelectedEditorImages] = useState([]); + const xfWsRef = useRef(null); + const xfAudioContextRef = useRef(null); + const xfMediaStreamRef = useRef(null); const [quickPrompts, setQuickPrompts] = useState([ '请完善报告内容', '请对内容做如下修改：' ]); @@ -85,6 +89,28 @@ export default function ReportEditor() { stateRef.current.chatMessages = chatMessages; }, [chatMessages]); + // 监听编辑器中已插入的图片，同步到 AI 面板 + useEffect(() => { + if (!editorRef.current) return; + const updateEditorImages = () => { + if (!editorRef.current) return; + const imgs = Array.from(editorRef.current.querySelectorAll('.image-placeholder.has-image img')) + .map((img, idx) => ({ id: `editor-img-${idx}-${(img as HTMLImageElement).src.slice(-16)}`, src: (img as HTMLImageElement).src })) + .filter(img => img.src); + setEditorImages(prev => { + const same = prev.length === imgs.length && prev.every((p, i) => p.src === imgs[i]?.src); + if (same) return prev; + // 清除已不存在的选中项 + setAiSelectedEditorImages(prevSelected => prevSelected.filter(id => imgs.some(img => img.id === id))); + return imgs; + }); + }; + updateEditorImages(); + const observer = new MutationObserver(updateEditorImages); + observer.observe(editorRef.current, { childList: true, subtree: true, attributes: true }); + return () => observer.disconnect(); + }, []); + useEffect(() => { stateRef.current.chatInput = chatInput; }, [chatInput]); @@ -853,44 +879,107 @@ export default function ReportEditor() { return html; }; - const toggleListening = () => { + async function getXfAuthUrl(apiKey: string, apiSecret: string): Promise { + const host = 'iat-api.xfyun.cn'; + const date = new Date().toUTCString(); + const signatureOrigin = `host: "${host}"\ndate: "${date}"\nGET /v2/iat HTTP/1.1`; + const encoder = new TextEncoder(); + const key = await crypto.subtle.importKey('raw', encoder.encode(apiSecret), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']); + const signature = await crypto.subtle.sign('HMAC', key, encoder.encode(signatureOrigin)); + const signatureBase64 = btoa(String.fromCharCode(...new Uint8Array(signature))); + const authorizationOrigin = `api_key="${apiKey}", algorithm="hmac-sha256", headers="host date request-line", signature="${signatureBase64}"`; + const authorization = btoa(authorizationOrigin); + return `wss://iat-api.xfyun.cn/v2/iat?authorization=${encodeURIComponent(authorization)}&date=${encodeURIComponent(date)}&host=${encodeURIComponent(host)}`; + } + + function floatTo16BitPCM(input: Float32Array): ArrayBuffer { + const output = new DataView(new ArrayBuffer(input.length * 2)); + for (let i = 0; i < input.length; i++) { + const s = Math.max(-1, Math.min(1, input[i])); + output.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7FFF, true); + } + return output.buffer; + } + + function arrayBufferToBase64(buffer: ArrayBuffer): string { + const bytes = new Uint8Array(buffer); + let binary = ''; + for (let i = 0; i < bytes.byteLength; i++) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); + } + + const toggleListening = async () => { if (isListening) { setIsListening(false); - if (speechRecognitionRef.current) speechRecognitionRef.current.stop(); - } else { - const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition; - if (!SpeechRecognition) { - alert('您的浏览器不支持原生语音识别，请使用 Chrome。'); - return; - } - const recognition = new SpeechRecognition(); - recognition.lang = 'zh-CN'; - recognition.continuous = true; - recognition.interimResults = true; - let finalTranscript = chatInput; - recognition.onstart = () => setIsListening(true); - recognition.onresult = (event: any) => { - let interimTranscript = ''; - for (let i = event.resultIndex; i < event.results.length; ++i) { - if (event.results[i].isFinal) { - finalTranscript += event.results[i][0].transcript; - } else { - interimTranscript += event.results[i][0].transcript; - } + if (xfWsRef.current) { try { xfWsRef.current.close(); } catch {} xfWsRef.current = null; } + if (xfAudioContextRef.current) { try { xfAudioContextRef.current.close(); } catch {} xfAudioContextRef.current = null; } + if (xfMediaStreamRef.current) { xfMediaStreamRef.current.getTracks().forEach(t => t.stop()); xfMediaStreamRef.current = null; } + return; + } + const xfConfig = storage.get('systemSettings', {} as SystemSettings).xfSpeechConfig; + if (!xfConfig?.appId || !xfConfig?.apiKey || !xfConfig?.apiSecret) { + alert('请先在系统设置中配置讯飞语音 APPID/APIKey/APISecret'); + return; + } + try { + const authUrl = await getXfAuthUrl(xfConfig.apiKey, xfConfig.apiSecret); + const ws = new WebSocket(authUrl); + xfWsRef.current = ws; + let frameStatus = 0; + let transcript = chatInput; + ws.onopen = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + xfMediaStreamRef.current = stream; + const audioContext = new AudioContext({ sampleRate: 16000 }); + xfAudioContextRef.current = audioContext; + const source = audioContext.createMediaStreamSource(stream); + const processor = audioContext.createScriptProcessor(4096, 1, 1); + processor.onaudioprocess = (e) => { + if (ws.readyState !== WebSocket.OPEN) return; + const inputData = e.inputBuffer.getChannelData(0); + const pcmBuffer = floatTo16BitPCM(inputData); + const base64Audio = arrayBufferToBase64(pcmBuffer); + const frame: any = { data: { status: frameStatus, format: 'audio/L16;rate=16000', encoding: 'raw', audio: base64Audio } }; + if (frameStatus === 0) { frame.common = { app_id: xfConfig.appId }; frame.business = { language: 'zh_cn', domain: 'iat', accent: 'mandarin' }; } + ws.send(JSON.stringify(frame)); + frameStatus = 1; + }; + source.connect(processor); + processor.connect(audioContext.destination); + setIsListening(true); + } catch (e: any) { + alert('麦克风启动失败: ' + e.message); + setIsListening(false); + ws.close(); } - setChatInput(finalTranscript + interimTranscript); }; - recognition.onerror = () => setIsListening(false); - recognition.onend = () => setIsListening(false); - speechRecognitionRef.current = recognition; - recognition.start(); + ws.onmessage = (event) => { + try { + const jsonData = JSON.parse(event.data); + if (jsonData.data?.result?.ws) { + let seg = ''; + for (const w of jsonData.data.result.ws) { if (w.cw?.[0]?.w) seg += w.cw[0].w; } + if (jsonData.data.result.ls) { transcript += seg; setChatInput(transcript); } + else { setChatInput(transcript + seg); } + } + } catch {} + }; + ws.onerror = () => { alert('讯飞语音连接失败'); setIsListening(false); }; + ws.onclose = () => { setIsListening(false); }; + } catch (e: any) { + alert('讯飞语音初始化失败: ' + e.message); } }; const handleAIGenerate = async (text: string) => { if (!text.trim()) return; const userMsgId = Date.now().toString(); - setChatMessages(prev => [...prev, { id: userMsgId, role: 'user', content: text }]); + const selectedEditorImageUrls = editorImages.filter(img => aiSelectedEditorImages.includes(img.id)).map(img => img.src); + const allImages = [...selectedEditorImageUrls, ...aiUploadedImages.map(i => i.dataUrl)]; + setChatMessages(prev => [...prev, { id: userMsgId, role: 'user', content: text, images: allImages.length > 0 ? allImages : undefined }]); setChatInput(''); setIsGenerating(true); try { @@ -933,8 +1022,8 @@ export default function ReportEditor() { const currentHtml = targetRegionEl ? targetRegionEl.innerHTML.replace(/​/g, '').replace(/>(\s+)<').trim() : ''; const globalContextText = editorRef.current?.innerText || ''; let messageContent: any; - const selectedFrameUrls = aiSelectedFrames.map(id => capturedFrames.find(f => f.id === id)?.dataUrl).filter(Boolean); - const allImages = [...selectedFrameUrls, ...aiUploadedImages.map(i => i.dataUrl)]; + const selectedEditorImageUrls = editorImages.filter(img => aiSelectedEditorImages.includes(img.id)).map(img => img.src); + const allImages = [...selectedEditorImageUrls, ...aiUploadedImages.map(i => i.dataUrl)]; let promptText = `【全局手术报告参考内容】:\n${globalContextText}\n\n`; if (aiModifyEnabled && targetRegionEl) { promptText += `【你需要进行修改的目标区域 HTML 源码】:\n${currentHtml || '（当前区域为空）'}\n\n`; @@ -1031,7 +1120,7 @@ export default function ReportEditor() { } } setAiUploadedImages([]); - setAiSelectedFrames([]); + setAiSelectedEditorImages([]); } catch (error: any) { console.error(error); setChatMessages(prev => [...prev, { id: Date.now().toString(), role: 'model', content: `【系统错误】: ${error.message}` }]); @@ -1243,7 +1332,7 @@ export default function ReportEditor() { setChatMessages([]); setChatInput(''); setAiUploadedImages([]); - setAiSelectedFrames([]); + setAiSelectedEditorImages([]); prevVideoCountRef.current = 0; stateRef.current = { ...stateRef.current, @@ -2331,7 +2420,14 @@ export default function ReportEditor() { chatMessages.map(msg => (
- {msg.content} +
{msg.content}
+ {msg.images && msg.images.length > 0 && ( +
+ {msg.images.map((src, idx) => ( + + ))} +
+ )}
)) @@ -2381,15 +2477,15 @@ export default function ReportEditor() { - {/* 视觉参考上下文 */} - {capturedFrames.length > 0 && ( + {/* 视觉参考上下文 - 编辑器中已插入的图片 */} + {editorImages.length > 0 && (
- {capturedFrames.map(frame => { - const isSelected = aiSelectedFrames.includes(frame.id); + {editorImages.map(img => { + const isSelected = aiSelectedEditorImages.includes(img.id); return ( -
setAiSelectedFrames(prev => isSelected ? prev.filter(id => id !== frame.id) : [...prev, frame.id])} +
setAiSelectedEditorImages(prev => isSelected ? prev.filter(id => id !== img.id) : [...prev, img.id])} className={`relative shrink-0 w-12 aspect-video rounded overflow-hidden border-2 cursor-pointer transition-all ${isSelected ? 'border-blue-600' : 'border-transparent opacity-50'}`}> - + {isSelected &&
}
); @@ -2435,7 +2531,7 @@ export default function ReportEditor() { modifyEnabled: aiModifyEnabled, chatInput, uploadedImagesCount: aiUploadedImages.length, - selectedFramesCount: aiSelectedFrames.length + selectedFramesCount: aiSelectedEditorImages.length } }; const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' }); diff --git a/src/pages/SystemSettings.tsx b/src/pages/SystemSettings.tsx index fd39c2b..51f9944 100644 --- a/src/pages/SystemSettings.tsx +++ b/src/pages/SystemSettings.tsx @@ -14,7 +14,8 @@ export default function SystemSettings() { defaultTemplate: '', frameMode: 'keep', activeAiProvider: 'kimi', - aiProviders: { ...DEFAULT_AI_PROVIDERS } + aiProviders: { ...DEFAULT_AI_PROVIDERS }, + xfIatConfig: { appId: 'e0fe23e3', apiKey: '7fd08be316718c2280e85af4fe126306', apiSecret: 'ZGI5MjAzZDA0YzYwNDhjMWZiNTM2NDE0' } }); const [templates, setTemplates] = useState([]); const [isSaved, setIsSaved] = useState(false); @@ -68,6 +69,7 @@ export default function SystemSettings() { if (!savedSettings.frameMode) savedSettings.frameMode = 'keep'; if (typeof savedSettings.autoInsertFrames !== 'boolean') savedSettings.autoInsertFrames = false; if (typeof savedSettings.autoInsertDelay !== 'number') savedSettings.autoInsertDelay = 0; + if (!savedSettings.xfSpeechConfig) savedSettings.xfSpeechConfig = { appId: 'e0fe23e3', apiKey: '7fd08be316718c2280e85af4fe126306', apiSecret: 'ZGI5MjAzZDA0YzYwNDhjMWZiNTM2NDE0' }; setSettings(savedSettings); } else if (savedTemplates.length > 0) { setSettings(prev => ({ ...prev, defaultTemplate: savedTemplates[0].id, frameMode: prev.frameMode || 'keep', autoInsertFrames: typeof prev.autoInsertFrames === 'boolean' ? prev.autoInsertFrames : false, autoInsertDelay: typeof prev.autoInsertDelay === 'number' ? prev.autoInsertDelay : 0 })); @@ -150,7 +152,8 @@ export default function SystemSettings() { aiProviders: { ...DEFAULT_AI_PROVIDERS, kimi: { ...DEFAULT_AI_PROVIDERS.kimi, apiKey: getDefaultApiKey() } }, autoInsertFrames: true, autoInsertDelay: 1, - autoInsertFrameIndices: [0, 2, 4, 6, 8, 10] + autoInsertFrameIndices: [0, 2, 4, 6, 8, 10], + xfSpeechConfig: { appId: 'e0fe23e3', apiKey: '7fd08be316718c2280e85af4fe126306', apiSecret: 'ZGI5MjAzZDA0YzYwNDhjMWZiNTM2NDE0' } }; setSettings(defaultSettings); storage.set('systemSettings', defaultSettings); @@ -421,6 +424,58 @@ export default function SystemSettings() {
)} + {currentUser.role === 'super' && ( +
+
+

+ + 讯飞语音配置 +

+
+
+
+ + { + const next = { ...(settings.xfIatConfig || { appId: '', apiKey: '', apiSecret: '' }), appId: e.target.value }; + setSettings({ ...settings, xfIatConfig: next }); + }} + placeholder="e0fe23e3" + className="input-minimal" + /> +
+
+ + { + const next = { ...(settings.xfIatConfig || { appId: '', apiKey: '', apiSecret: '' }), apiKey: e.target.value }; + setSettings({ ...settings, xfIatConfig: next }); + }} + placeholder="7fd08be316718c2280e85af4fe126306" + className="input-minimal" + /> +
+
+ + { + const next = { ...(settings.xfIatConfig || { appId: '', apiKey: '', apiSecret: '' }), apiSecret: e.target.value }; + setSettings({ ...settings, xfIatConfig: next }); + }} + placeholder="ZGI5MjAzZDA0YzYwNDhjMWZiNTM2NDE0" + className="input-minimal" + /> +
+
+
+ )} +

diff --git a/src/types.ts b/src/types.ts index 43d63bc..650a7d1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -86,6 +86,7 @@ export interface SystemSettings { autoInsertDelay?: number; activeAiProvider: string; aiProviders: Record; + xfSpeechConfig?: { appId: string; apiKey: string; apiSecret: string }; } export const DEFAULT_AI_PROVIDERS: Record = {