2184 lines
93 KiB
Python
Executable File
2184 lines
93 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
批量处理住院病案首页 PDF。
|
||
|
||
默认从项目根目录的“待处理-患者首页PDF”读取 PDF，并把 CSV/JSONL/单份 JSON
|
||
写入“数据处理结果区”。脚本只依赖 Python 标准库和系统命令 pdftotext。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import json
|
||
import os
|
||
import re
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
|
||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||
DEFAULT_INPUT_DIR = PROJECT_ROOT / "待处理-患者首页PDF"
|
||
DEFAULT_OUTPUT_DIR = PROJECT_ROOT / "数据处理结果区"
|
||
DEFAULT_PG_TABLE = os.environ.get("PGTABLE") or os.environ.get("PG_PATIENT_TABLE", "")
|
||
DEFAULT_DEPARTMENT_RULE_PATH = PROJECT_ROOT / "数据处理工作区" / "01_配置规则" / "01_科室分类规则.json"
|
||
DEFAULT_MINERU_CLIENT_PATH = PROJECT_ROOT / "数据处理工作区" / "05_备用读取" / "05_备用PDF转Markdown_Mineru.py"
|
||
DEFAULT_MINERU_MD_DIR = DEFAULT_OUTPUT_DIR / "06_Mineru_MD"
|
||
DEFAULT_MINERU_URL = os.environ.get("MINERU_URL", "")
|
||
|
||
|
||
CSV_COLUMNS = [
|
||
"住院号",
|
||
"源文件",
|
||
"病案号",
|
||
"首页病案号",
|
||
"姓名",
|
||
"性别",
|
||
"出生日期",
|
||
"年龄",
|
||
"身份证号",
|
||
"新生儿年龄（月）",
|
||
"新生儿出生体重（克）",
|
||
"新生儿入院体重（克）",
|
||
"医疗机构",
|
||
"组织机构代码",
|
||
"医疗付费方式",
|
||
"住院次数",
|
||
"入院时间",
|
||
"入院科别",
|
||
"入院病房",
|
||
"转科科别",
|
||
"转科时间",
|
||
"出院时间",
|
||
"出院科别",
|
||
"出院病房",
|
||
"大科室",
|
||
"实际住院天数",
|
||
"门急诊诊断",
|
||
"门急诊诊断编码",
|
||
"主要诊断",
|
||
"主要诊断编码",
|
||
"主要诊断入院病情",
|
||
"出院诊断",
|
||
"手术操作",
|
||
"病理诊断",
|
||
"病理诊断编码",
|
||
"病理号",
|
||
"药物过敏代码",
|
||
"过敏药物",
|
||
"死亡患者尸检代码",
|
||
"血型代码",
|
||
"Rh代码",
|
||
"科主任",
|
||
"主任副主任医师",
|
||
"主治医师",
|
||
"住院医师",
|
||
"责任护士",
|
||
"进修医师",
|
||
"实习医师",
|
||
"规培医师",
|
||
"编码员",
|
||
"病案质量代码",
|
||
"质控医师",
|
||
"质控护士",
|
||
"质控日期",
|
||
"离院方式代码",
|
||
"出院31天内再住院计划代码",
|
||
"总费用",
|
||
"自付金额",
|
||
"质控状态",
|
||
"质控提示",
|
||
"复核状态",
|
||
"复核备注",
|
||
"文本抽取方式",
|
||
"自动修正",
|
||
"人工修正",
|
||
]
|
||
|
||
|
||
PG_COLUMNS: list[tuple[str, str]] = [
|
||
("source_file", "TEXT NOT NULL"),
|
||
("inpatient_no", "TEXT"),
|
||
("medical_record_no", "TEXT"),
|
||
("front_page_medical_record_no", "TEXT"),
|
||
("patient_name", "TEXT"),
|
||
("gender", "TEXT"),
|
||
("birth_date", "DATE"),
|
||
("age", "TEXT"),
|
||
("nationality", "TEXT"),
|
||
("id_card_no", "TEXT"),
|
||
("neonatal_age_months", "INTEGER"),
|
||
("newborn_birth_weight_g", "INTEGER"),
|
||
("newborn_admission_weight_g", "INTEGER"),
|
||
("hospital_name", "TEXT"),
|
||
("organization_code", "TEXT"),
|
||
("payment_method", "TEXT"),
|
||
("health_card_no", "TEXT"),
|
||
("admission_count", "INTEGER"),
|
||
("birthplace", "TEXT"),
|
||
("native_place", "TEXT"),
|
||
("ethnicity", "TEXT"),
|
||
("occupation", "TEXT"),
|
||
("marital_status_code", "TEXT"),
|
||
("current_address", "TEXT"),
|
||
("current_address_phone", "TEXT"),
|
||
("current_address_postcode", "TEXT"),
|
||
("household_address", "TEXT"),
|
||
("household_postcode", "TEXT"),
|
||
("employer_address", "TEXT"),
|
||
("employer_phone", "TEXT"),
|
||
("employer_postcode", "TEXT"),
|
||
("contact_name", "TEXT"),
|
||
("contact_relationship", "TEXT"),
|
||
("contact_address", "TEXT"),
|
||
("contact_phone", "TEXT"),
|
||
("admission_path_code", "TEXT"),
|
||
("admission_time", "TIMESTAMP"),
|
||
("admission_dept", "TEXT"),
|
||
("admission_ward", "TEXT"),
|
||
("transfer_dept", "TEXT"),
|
||
("transfer_time", "TEXT"),
|
||
("discharge_time", "TIMESTAMP"),
|
||
("discharge_dept", "TEXT"),
|
||
("discharge_ward", "TEXT"),
|
||
("major_department", "TEXT"),
|
||
("hospital_days", "INTEGER"),
|
||
("outpatient_diagnosis", "TEXT"),
|
||
("outpatient_diagnosis_code", "TEXT"),
|
||
("primary_diagnosis", "TEXT"),
|
||
("primary_diagnosis_code", "TEXT"),
|
||
("primary_admission_condition", "TEXT"),
|
||
("discharge_diagnoses", "JSONB"),
|
||
("operations", "JSONB"),
|
||
("injury_poisoning_external_cause", "TEXT"),
|
||
("injury_poisoning_code", "TEXT"),
|
||
("pathology_diagnosis", "TEXT"),
|
||
("pathology_diagnosis_code", "TEXT"),
|
||
("pathology_no", "TEXT"),
|
||
("drug_allergy_code", "TEXT"),
|
||
("allergy_drug", "TEXT"),
|
||
("autopsy_code", "TEXT"),
|
||
("blood_type_code", "TEXT"),
|
||
("rh_code", "TEXT"),
|
||
("department_director", "TEXT"),
|
||
("chief_physician", "TEXT"),
|
||
("attending_physician", "TEXT"),
|
||
("resident_physician", "TEXT"),
|
||
("responsible_nurse", "TEXT"),
|
||
("refresher_physician", "TEXT"),
|
||
("intern_physician", "TEXT"),
|
||
("standardized_resident_physician", "TEXT"),
|
||
("coder", "TEXT"),
|
||
("record_quality_code", "TEXT"),
|
||
("quality_control_physician", "TEXT"),
|
||
("quality_control_nurse", "TEXT"),
|
||
("quality_control_date", "DATE"),
|
||
("discharge_disposition_code", "TEXT"),
|
||
("receiving_org_name", "TEXT"),
|
||
("readmission_plan_code", "TEXT"),
|
||
("readmission_plan_purpose", "TEXT"),
|
||
("coma_before_days", "INTEGER"),
|
||
("coma_before_hours", "INTEGER"),
|
||
("coma_before_minutes", "INTEGER"),
|
||
("coma_after_days", "INTEGER"),
|
||
("coma_after_hours", "INTEGER"),
|
||
("coma_after_minutes", "INTEGER"),
|
||
("total_cost", "NUMERIC(12,2)"),
|
||
("self_pay_amount", "NUMERIC(12,2)"),
|
||
("fee_details", "JSONB"),
|
||
("quality_status", "TEXT"),
|
||
("quality_notes", "JSONB"),
|
||
("review_status", "TEXT NOT NULL DEFAULT 'pending'"),
|
||
("review_notes", "JSONB NOT NULL DEFAULT '[]'::jsonb"),
|
||
("manual_corrected", "BOOLEAN NOT NULL DEFAULT false"),
|
||
("auto_corrections", "JSONB NOT NULL DEFAULT '[]'::jsonb"),
|
||
("text_extraction_method", "TEXT"),
|
||
("mineru_markdown_dir", "TEXT"),
|
||
("raw_text", "TEXT"),
|
||
]
|
||
|
||
|
||
PG_COLUMN_COMMENTS: dict[str, str] = {
|
||
"id": "自增主键，仅用于数据库内部定位记录。",
|
||
"source_file": "来源PDF文件名；重复入库时以住院号为准更新。",
|
||
"inpatient_no": "患者号/住院号，作为首页与患者列表联动唯一键；不能为空，格式由患者目录核验端处理。",
|
||
"medical_record_no": "病案号，统一保存为10位文本，保留前导0。",
|
||
"front_page_medical_record_no": "PDF首页病案号，统一保存为10位文本，保留前导0。",
|
||
"patient_name": "患者姓名。",
|
||
"gender": "患者性别。",
|
||
"birth_date": "出生日期。",
|
||
"age": "首页记录的住院年龄。",
|
||
"nationality": "国籍。",
|
||
"id_card_no": "居民身份证号。",
|
||
"neonatal_age_months": "年龄不足1周岁患儿的年龄（月）。",
|
||
"newborn_birth_weight_g": "新生儿出生体重（克）。",
|
||
"newborn_admission_weight_g": "新生儿入院体重（克）。",
|
||
"hospital_name": "医疗机构名称。",
|
||
"organization_code": "医疗机构组织机构代码。",
|
||
"payment_method": "医疗付费方式。",
|
||
"health_card_no": "健康卡号。",
|
||
"admission_count": "本机构第几次住院。",
|
||
"birthplace": "出生地。",
|
||
"native_place": "籍贯。",
|
||
"ethnicity": "民族。",
|
||
"occupation": "职业。",
|
||
"marital_status_code": "婚姻状况代码：1未婚、2已婚、3丧偶、4离婚、9其他。",
|
||
"current_address": "现住址。",
|
||
"current_address_phone": "现住址联系电话。",
|
||
"current_address_postcode": "现住址邮编。",
|
||
"household_address": "户口地址。",
|
||
"household_postcode": "户口地址邮编。",
|
||
"employer_address": "工作单位及地址。",
|
||
"employer_phone": "单位电话。",
|
||
"employer_postcode": "单位邮编。",
|
||
"contact_name": "联系人姓名，位于首页第一面“联系人姓名”栏。",
|
||
"contact_relationship": "联系人与患者关系，位于首页第一面“关系”栏。",
|
||
"contact_address": "联系人地址，位于首页第一面“联系人姓名/关系/地址/电话”这一行的“地址”栏；不是入院途径选项。",
|
||
"contact_phone": "联系人电话，位于首页第一面“电话”栏。",
|
||
"admission_path_code": "入院途径代码：1急诊、2门诊、3其他医疗机构转入、9其他。",
|
||
"admission_time": "入院时间。",
|
||
"admission_dept": "入院科别。",
|
||
"admission_ward": "入院病房。",
|
||
"transfer_dept": "转科科别。",
|
||
"transfer_time": "转科时间；首页该项常为空或为横线。",
|
||
"discharge_time": "出院时间。",
|
||
"discharge_dept": "出院科别。",
|
||
"discharge_ward": "出院病房。",
|
||
"major_department": "大科室分类，由科室分类规则根据出院科别优先、入院科别兜底映射。",
|
||
"hospital_days": "实际住院天数。",
|
||
"outpatient_diagnosis": "门（急）诊诊断。",
|
||
"outpatient_diagnosis_code": "门（急）诊诊断疾病编码。",
|
||
"primary_diagnosis": "主要出院诊断名称。",
|
||
"primary_diagnosis_code": "主要出院诊断疾病编码。",
|
||
"primary_admission_condition": "主要诊断入院病情代码。",
|
||
"discharge_diagnoses": "出院诊断明细JSON数组，包含主要诊断和其他诊断。",
|
||
"operations": "手术及操作明细JSON数组。",
|
||
"injury_poisoning_external_cause": "损伤、中毒的外部原因。",
|
||
"injury_poisoning_code": "损伤、中毒外部原因疾病编码。",
|
||
"pathology_diagnosis": "病理诊断。",
|
||
"pathology_diagnosis_code": "病理诊断疾病编码。",
|
||
"pathology_no": "病理号。",
|
||
"drug_allergy_code": "药物过敏代码：1无、2有。",
|
||
"allergy_drug": "过敏药物名称。",
|
||
"autopsy_code": "死亡患者尸检代码：1是、2否、3-。",
|
||
"blood_type_code": "ABO血型代码：1A、2B、3O、4AB、5不详、6未查。",
|
||
"rh_code": "Rh血型代码：1阴、2阳、3不详、4未查。",
|
||
"department_director": "科主任。",
|
||
"chief_physician": "主任（副主任）医师。",
|
||
"attending_physician": "主治医师。",
|
||
"resident_physician": "住院医师。",
|
||
"responsible_nurse": "责任护士。",
|
||
"refresher_physician": "进修医师。",
|
||
"intern_physician": "实习医师。",
|
||
"standardized_resident_physician": "规培医师。",
|
||
"coder": "病案首页编码员。",
|
||
"record_quality_code": "病案质量代码：1甲、2乙、3丙。",
|
||
"quality_control_physician": "质控医师。",
|
||
"quality_control_nurse": "质控护士。",
|
||
"quality_control_date": "质控日期。",
|
||
"discharge_disposition_code": "离院方式代码：1医嘱离院、2医嘱转院、3医嘱转社区/乡镇卫生院、4非医嘱离院、5死亡、9其他。",
|
||
"receiving_org_name": "拟接收医疗机构名称。",
|
||
"readmission_plan_code": "是否有出院31天内再住院计划代码：1无、2有。",
|
||
"readmission_plan_purpose": "出院31天内再住院计划目的。",
|
||
"coma_before_days": "颅脑损伤患者昏迷时间：入院前天数。",
|
||
"coma_before_hours": "颅脑损伤患者昏迷时间：入院前小时数。",
|
||
"coma_before_minutes": "颅脑损伤患者昏迷时间：入院前分钟数。",
|
||
"coma_after_days": "颅脑损伤患者昏迷时间：入院后天数。",
|
||
"coma_after_hours": "颅脑损伤患者昏迷时间：入院后小时数。",
|
||
"coma_after_minutes": "颅脑损伤患者昏迷时间：入院后分钟数。",
|
||
"total_cost": "住院总费用。",
|
||
"self_pay_amount": "自付金额。",
|
||
"fee_details": "住院费用分类明细JSON对象。",
|
||
"quality_status": "程序质控状态。",
|
||
"quality_notes": "程序质控提示JSON数组。",
|
||
"review_status": "复核状态：auto_pass自动通过、auto_corrected已自动修正、needs_review需复核、reviewed已人工复核。",
|
||
"review_notes": "人工或程序复核备注JSON数组。",
|
||
"manual_corrected": "是否经过人工修正。",
|
||
"auto_corrections": "程序自动修正记录JSON数组。",
|
||
"text_extraction_method": "本次解析使用的文本抽取方式：pdftotext或mineru_markdown。",
|
||
"mineru_markdown_dir": "Mineru Markdown输出目录；未使用Mineru时为空。",
|
||
"raw_text": "PDF抽取出的首页原始文本，可能来自pdftotext或Mineru Markdown，用于追溯和人工核对。",
|
||
}
|
||
|
||
|
||
def normalize_spaces(text: str) -> str:
|
||
return re.sub(r"[ \t]+", " ", text.strip())
|
||
|
||
|
||
def clean_value(value: str | None) -> str:
|
||
if value is None:
|
||
return ""
|
||
value = normalize_spaces(value)
|
||
return "" if value in {"-", "—", "无"} else value
|
||
|
||
|
||
def clean_int_value(value: str | None) -> str:
|
||
value = clean_value(value)
|
||
return value if re.fullmatch(r"\d+", value) else ""
|
||
|
||
|
||
def first_match(pattern: str, text: str, group: int = 1, flags: int = 0) -> str:
|
||
match = re.search(pattern, text, flags)
|
||
if not match:
|
||
return ""
|
||
return clean_value(match.group(group))
|
||
|
||
|
||
def filename_medical_record_no(pdf_path: Path) -> str:
|
||
match = re.match(r"^ZY\d{2}(\d{10})", pdf_path.stem, flags=re.IGNORECASE)
|
||
return match.group(1) if match else ""
|
||
|
||
|
||
def filename_admission_count(pdf_path: Path) -> str:
|
||
match = re.match(r"^ZY(\d{2})\d{10}", pdf_path.stem, flags=re.IGNORECASE)
|
||
return match.group(1) if match else ""
|
||
|
||
|
||
def filename_inpatient_no(pdf_path: Path) -> str:
|
||
match = re.match(r"^(ZY\d{12})", pdf_path.stem, flags=re.IGNORECASE)
|
||
return match.group(1).upper() if match else ""
|
||
|
||
|
||
def normalize_digits(value: Any, width: int) -> str:
|
||
digits = re.sub(r"\D", "", clean_value(str(value)) if value is not None else "")
|
||
if not digits:
|
||
return ""
|
||
return digits[-width:].zfill(width)
|
||
|
||
|
||
def build_inpatient_no(
|
||
admission_count: Any,
|
||
front_page_medical_record_no: Any,
|
||
medical_record_no: Any,
|
||
pdf_path: Path,
|
||
) -> tuple[str, list[str], list[str]]:
|
||
corrections: list[str] = []
|
||
notes: list[str] = []
|
||
filename_no = filename_inpatient_no(pdf_path)
|
||
|
||
admission = normalize_digits(admission_count, 2) or filename_admission_count(pdf_path)
|
||
page_no = normalize_digits(front_page_medical_record_no, 10) or normalize_digits(medical_record_no, 10) or filename_medical_record_no(pdf_path)
|
||
if admission and page_no:
|
||
inpatient_no = f"ZY{admission}{page_no}"
|
||
if filename_no and filename_no != inpatient_no:
|
||
notes.append(f"住院号{inpatient_no}与文件名住院号{filename_no}不一致，请核对住院次数和首页病案号")
|
||
if clean_value(str(front_page_medical_record_no)) and normalize_digits(front_page_medical_record_no, 10) != clean_value(str(front_page_medical_record_no)):
|
||
corrections.append(f"首页病案号用于住院号时补齐为{page_no}")
|
||
return inpatient_no, corrections, notes
|
||
|
||
if filename_no:
|
||
corrections.append(f"住院号由文件名补充为{filename_no}")
|
||
return filename_no, corrections, notes
|
||
|
||
notes.append("住院号无法生成：缺少住院次数或首页病案号")
|
||
return "", corrections, notes
|
||
|
||
|
||
def normalize_medical_record_no(raw_no: str, pdf_path: Path) -> tuple[str, str, list[str], list[str]]:
|
||
original = clean_value(raw_no)
|
||
from_filename = filename_medical_record_no(pdf_path)
|
||
corrections: list[str] = []
|
||
notes: list[str] = []
|
||
|
||
if from_filename and original != from_filename:
|
||
if original and from_filename.endswith(original):
|
||
corrections.append(f"病案号由PDF值{original}按文件名补齐为{from_filename}")
|
||
elif original:
|
||
notes.append(f"PDF病案号{original}与文件名病案号{from_filename}不一致，已优先采用文件名")
|
||
else:
|
||
corrections.append(f"病案号由文件名补充为{from_filename}")
|
||
return from_filename, original, corrections, notes
|
||
|
||
if re.fullmatch(r"\d{1,9}", original):
|
||
normalized = original.zfill(10)
|
||
corrections.append(f"病案号由{original}补齐为{normalized}")
|
||
return normalized, original, corrections, notes
|
||
|
||
return original, original, corrections, notes
|
||
|
||
|
||
def normalize_department_key(value: str) -> str:
|
||
return re.sub(r"\s+", "", clean_value(value))
|
||
|
||
|
||
def department_lookup_candidates(value: str) -> list[str]:
|
||
base = normalize_department_key(value)
|
||
candidates: list[str] = []
|
||
|
||
def add(candidate: str) -> None:
|
||
candidate = normalize_department_key(candidate)
|
||
if candidate and candidate not in candidates:
|
||
candidates.append(candidate)
|
||
|
||
add(base)
|
||
add(base.replace("病房", ""))
|
||
add(base.replace("病区", ""))
|
||
add(base.replace("病房", "").replace("病区", ""))
|
||
|
||
for candidate in list(candidates):
|
||
add(candidate.replace("胸外科", "胸外"))
|
||
add(candidate.replace("泌尿外科", "泌尿外"))
|
||
add(candidate.replace("感染科", "感染"))
|
||
add(candidate.replace("普通外科", "普外科"))
|
||
|
||
return candidates
|
||
|
||
|
||
def load_department_rules(rule_path: Path) -> dict[str, dict[str, str]]:
|
||
if not rule_path.exists():
|
||
return {}
|
||
data = json.loads(rule_path.read_text(encoding="utf-8"))
|
||
aliases = {
|
||
normalize_department_key(alias): normalize_department_key(standard)
|
||
for alias, standard in data.get("aliases", {}).items()
|
||
}
|
||
standard_to_major: dict[str, str] = {}
|
||
for group in data.get("大科室列表", []):
|
||
major = clean_value(group.get("大科室", ""))
|
||
for department in group.get("子科室", []):
|
||
standard = normalize_department_key(department)
|
||
standard_to_major[standard] = major
|
||
aliases.setdefault(standard, standard)
|
||
return {"aliases": aliases, "standard_to_major": standard_to_major}
|
||
|
||
|
||
def classify_major_department(record: dict[str, Any], rules: dict[str, dict[str, str]]) -> tuple[str, str]:
|
||
if not rules:
|
||
return "", ""
|
||
aliases = rules.get("aliases", {})
|
||
standard_to_major = rules.get("standard_to_major", {})
|
||
for source_key in ["出院科别", "入院科别"]:
|
||
for candidate in department_lookup_candidates(str(record.get(source_key, ""))):
|
||
standard = aliases.get(candidate, candidate)
|
||
major = standard_to_major.get(standard)
|
||
if major:
|
||
return major, standard
|
||
return "", ""
|
||
|
||
|
||
def extract_text_with_pdftotext(pdf_path: Path) -> str:
|
||
if shutil.which("pdftotext") is None:
|
||
raise RuntimeError("未找到 pdftotext。请先安装 poppler-utils 后再运行。")
|
||
|
||
command = ["pdftotext", "-layout", "-enc", "UTF-8", str(pdf_path), "-"]
|
||
completed = subprocess.run(
|
||
command,
|
||
check=False,
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
text=True,
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
)
|
||
if completed.returncode != 0:
|
||
raise RuntimeError(completed.stderr.strip() or "pdftotext 转换失败")
|
||
return completed.stdout.replace("\r\n", "\n").replace("\r", "\n")
|
||
|
||
|
||
def text_needs_mineru_fallback(text: str) -> bool:
|
||
stripped = text.strip()
|
||
if len(stripped) < 200:
|
||
return True
|
||
required_markers = ["病案号", "医疗机构", "主要诊断"]
|
||
return sum(1 for marker in required_markers if marker in stripped) < 2
|
||
|
||
|
||
def markdown_files_for_pdf(pdf_path: Path, md_dir: Path) -> list[Path]:
|
||
folder = md_dir / pdf_path.stem
|
||
if not folder.exists():
|
||
return []
|
||
return sorted([*folder.rglob("*.md"), *folder.rglob("*.markdown"), *folder.rglob("*.txt")])
|
||
|
||
|
||
def read_mineru_markdown(pdf_path: Path, md_dir: Path) -> str:
|
||
parts: list[str] = []
|
||
for file_path in markdown_files_for_pdf(pdf_path, md_dir):
|
||
parts.append(file_path.read_text(encoding="utf-8", errors="replace"))
|
||
return "\n\n".join(parts).replace("\r\n", "\n").replace("\r", "\n")
|
||
|
||
|
||
def ensure_mineru_markdown(pdf_path: Path, args: argparse.Namespace) -> None:
|
||
md_dir = args.mineru_md_dir.resolve()
|
||
if markdown_files_for_pdf(pdf_path, md_dir):
|
||
return
|
||
if not args.mineru_url:
|
||
raise RuntimeError("缺少 Mineru API 服务地址：请设置 MINERU_URL 或传入 --mineru-url")
|
||
client_path = args.mineru_client.resolve()
|
||
if not client_path.exists():
|
||
raise RuntimeError(f"未找到 Mineru 客户端：{client_path}")
|
||
|
||
command = [
|
||
sys.executable,
|
||
str(client_path),
|
||
"-s",
|
||
str(pdf_path.parent.resolve()),
|
||
"-t",
|
||
str(md_dir),
|
||
"-u",
|
||
args.mineru_url,
|
||
]
|
||
if args.mineru_sync:
|
||
command.append("--sync")
|
||
completed = subprocess.run(
|
||
command,
|
||
check=False,
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
text=True,
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
)
|
||
if completed.returncode != 0:
|
||
raise RuntimeError(completed.stderr.strip() or completed.stdout.strip() or "Mineru PDF转Markdown失败")
|
||
|
||
|
||
def extract_record_text(pdf_path: Path, args: argparse.Namespace) -> tuple[str, str, str]:
|
||
text_source = args.text_source
|
||
if text_source == "pdftotext":
|
||
return extract_text_with_pdftotext(pdf_path), "pdftotext", ""
|
||
|
||
if text_source == "mineru":
|
||
ensure_mineru_markdown(pdf_path, args)
|
||
markdown_text = read_mineru_markdown(pdf_path, args.mineru_md_dir.resolve())
|
||
if not markdown_text.strip():
|
||
raise RuntimeError(f"Mineru未生成可读取的Markdown：{pdf_path.name}")
|
||
return markdown_text, "mineru_markdown", str((args.mineru_md_dir.resolve() / pdf_path.stem))
|
||
|
||
try:
|
||
pdftotext_text = extract_text_with_pdftotext(pdf_path)
|
||
except Exception:
|
||
pdftotext_text = ""
|
||
if pdftotext_text and not text_needs_mineru_fallback(pdftotext_text):
|
||
return pdftotext_text, "pdftotext", ""
|
||
|
||
try:
|
||
ensure_mineru_markdown(pdf_path, args)
|
||
markdown_text = read_mineru_markdown(pdf_path, args.mineru_md_dir.resolve())
|
||
if markdown_text.strip() and not text_needs_mineru_fallback(markdown_text):
|
||
return markdown_text, "mineru_markdown", str((args.mineru_md_dir.resolve() / pdf_path.stem))
|
||
except Exception as exc:
|
||
if pdftotext_text:
|
||
return pdftotext_text, f"pdftotext_mineru_failed:{exc}", ""
|
||
raise
|
||
|
||
if pdftotext_text:
|
||
return pdftotext_text, "pdftotext_suspicious_mineru_unusable", ""
|
||
return "", "empty", ""
|
||
|
||
|
||
def meaningful_lines(text: str) -> list[str]:
|
||
lines: list[str] = []
|
||
for raw in text.splitlines():
|
||
line = raw.strip()
|
||
if line:
|
||
lines.append(line)
|
||
return lines
|
||
|
||
|
||
def parse_header(text: str, lines: list[str]) -> dict[str, Any]:
|
||
result: dict[str, Any] = {
|
||
"组织机构代码": first_match(r"组织机构代码:\s*([^)）]+)", text),
|
||
"医疗机构": first_match(r"医疗机构:\s*(.+)", text),
|
||
"医疗付费方式": first_match(r"医疗付费方式:\s*(.+)", text),
|
||
"健康卡号": first_match(r"健康卡号:\s*(.*?)\s*第", text),
|
||
"住院次数": first_match(r"第\s*(\d+)\s*次住院", text),
|
||
"病案号": first_match(r"病案号:\s*([A-Za-z0-9]+)", text),
|
||
}
|
||
|
||
for idx, line in enumerate(lines):
|
||
if "病案号:" not in line or idx + 1 >= len(lines):
|
||
continue
|
||
patient_line = lines[idx + 1]
|
||
match = re.search(
|
||
r"^\s*(?P<name>\S+)\s+(?P<sex>男|女)\s+"
|
||
r"(?P<year>\d{4})\s*年\s*(?P<month>\d{1,2})\s*月\s*"
|
||
r"(?P<day>\d{1,2})\s*日\s+(?P<age>[\d.]+)\s+(?P<nation>\S+)",
|
||
patient_line,
|
||
)
|
||
if match:
|
||
result.update(
|
||
{
|
||
"姓名": clean_value(match.group("name")),
|
||
"性别": match.group("sex"),
|
||
"出生日期": f"{int(match.group('year')):04d}-{int(match.group('month')):02d}-{int(match.group('day')):02d}",
|
||
"年龄": clean_value(match.group("age")),
|
||
"国籍": clean_value(match.group("nation")),
|
||
}
|
||
)
|
||
break
|
||
|
||
return result
|
||
|
||
|
||
def parse_newborn_info(lines: list[str]) -> dict[str, str]:
|
||
result = {
|
||
"新生儿年龄（月）": "",
|
||
"新生儿出生体重（克）": "",
|
||
"新生儿入院体重（克）": "",
|
||
}
|
||
for idx, line in enumerate(lines):
|
||
if "年龄不足1周岁" not in line:
|
||
continue
|
||
joined = line + " " + (lines[idx + 1] if idx + 1 < len(lines) else "")
|
||
result["新生儿年龄（月）"] = clean_int_value(first_match(r"）\s*([-\d]+)\s*月", joined))
|
||
weights = [clean_int_value(item) for item in re.findall(r"([-\d]+)\s*克", joined)]
|
||
weights = [item for item in weights if item]
|
||
if weights:
|
||
result["新生儿出生体重（克）"] = weights[0]
|
||
result["新生儿入院体重（克）"] = weights[-1]
|
||
break
|
||
return result
|
||
|
||
|
||
def parse_basic_lines(lines: list[str]) -> dict[str, Any]:
|
||
result: dict[str, Any] = {}
|
||
id_index = -1
|
||
|
||
for idx, line in enumerate(lines):
|
||
match = re.match(r"^([1-9]\d{5}(?:18|19|20)\d{2}\d{2}\d{2}\d{3}[\dXx]|-)\s+(.+?)\s+([1-9])\s+1\.未婚", line)
|
||
if match:
|
||
id_index = idx
|
||
result["身份证号"] = clean_value(match.group(1)).upper()
|
||
result["职业"] = clean_value(match.group(2))
|
||
result["婚姻代码"] = clean_value(match.group(3))
|
||
if idx > 0:
|
||
parts = re.split(r"\s{2,}", lines[idx - 1].strip())
|
||
if len(parts) >= 3:
|
||
result["出生地"] = clean_value(parts[0])
|
||
result["籍贯"] = clean_value(parts[1])
|
||
result["民族"] = clean_value(parts[-1])
|
||
break
|
||
|
||
if id_index >= 0:
|
||
address_names = [
|
||
("现住址", "现住址电话", "现住址邮编"),
|
||
("户口地址", "", "户口地址邮编"),
|
||
("工作单位及地址", "单位电话", "单位邮编"),
|
||
]
|
||
for offset, names in enumerate(address_names, start=1):
|
||
if id_index + offset >= len(lines):
|
||
continue
|
||
address, phone, postcode = split_address_phone_postcode(lines[id_index + offset])
|
||
result[names[0]] = address
|
||
if names[1]:
|
||
result[names[1]] = phone
|
||
result[names[2]] = postcode
|
||
|
||
contact_line = find_contact_line(lines, id_index + 4)
|
||
if contact_line:
|
||
contact = parse_contact_line(contact_line)
|
||
result.update(contact)
|
||
|
||
return result
|
||
|
||
|
||
def split_address_phone_postcode(line: str) -> tuple[str, str, str]:
|
||
pieces = re.split(r"\s{2,}", line.strip())
|
||
if not pieces:
|
||
return "", "", ""
|
||
postcode = ""
|
||
phone = ""
|
||
if pieces and re.fullmatch(r"[\d-]{5,}|-", pieces[-1]):
|
||
postcode = clean_value(pieces.pop())
|
||
if pieces and re.fullmatch(r"[\d-]{7,}|-", pieces[-1]):
|
||
phone = clean_value(pieces.pop())
|
||
return clean_value(" ".join(pieces)), phone, postcode
|
||
|
||
|
||
def parse_contact_line(line: str) -> dict[str, str]:
|
||
if is_admission_path_line(line):
|
||
return {
|
||
"联系人姓名": "",
|
||
"联系人关系": "",
|
||
"联系人地址": "",
|
||
"联系人电话": "",
|
||
}
|
||
pieces = [clean_value(p) for p in re.split(r"\s{2,}", line.strip()) if clean_value(p)]
|
||
result = {
|
||
"联系人姓名": "",
|
||
"联系人关系": "",
|
||
"联系人地址": "",
|
||
"联系人电话": "",
|
||
}
|
||
if len(pieces) >= 1:
|
||
result["联系人姓名"] = pieces[0]
|
||
if len(pieces) >= 2:
|
||
result["联系人关系"] = pieces[1]
|
||
if len(pieces) >= 3:
|
||
if re.fullmatch(r"\d{7,}", pieces[-1]):
|
||
result["联系人电话"] = pieces[-1]
|
||
result["联系人地址"] = clean_value(" ".join(pieces[2:-1]))
|
||
else:
|
||
result["联系人地址"] = clean_value(" ".join(pieces[2:]))
|
||
return result
|
||
|
||
|
||
def is_admission_path_line(line: str) -> bool:
|
||
return bool(re.search(r"1\.急诊\s+2\.门诊", line)) or "其他医疗机构转入" in line
|
||
|
||
|
||
def find_contact_line(lines: list[str], start_index: int) -> str:
|
||
for idx in range(start_index, min(start_index + 3, len(lines))):
|
||
line = lines[idx]
|
||
if is_admission_path_line(line):
|
||
return ""
|
||
if re.search(r"\d{4}\s*年\s*\d{1,2}\s*月", line):
|
||
return ""
|
||
pieces = [clean_value(p) for p in re.split(r"\s{2,}", line.strip()) if clean_value(p)]
|
||
if len(pieces) >= 2:
|
||
return line
|
||
return ""
|
||
|
||
|
||
def parse_admission_discharge(lines: list[str]) -> dict[str, Any]:
|
||
result: dict[str, Any] = {}
|
||
date_line_indexes: list[int] = []
|
||
|
||
for idx, line in enumerate(lines):
|
||
if re.search(r"\d{4}\s*年\s*\d{1,2}\s*月\s*\d{1,2}\s*日\s*\d{1,2}\s*时", line):
|
||
date_line_indexes.append(idx)
|
||
|
||
if date_line_indexes:
|
||
result.update(parse_visit_line(lines[date_line_indexes[0]], prefix="入院"))
|
||
if date_line_indexes[0] + 1 < len(lines):
|
||
result["转科科别"] = clean_value(lines[date_line_indexes[0] + 1])
|
||
if date_line_indexes[0] + 2 < len(lines):
|
||
result["转科时间"] = clean_value(lines[date_line_indexes[0] + 2])
|
||
if len(date_line_indexes) >= 2:
|
||
result.update(parse_visit_line(lines[date_line_indexes[1]], prefix="出院"))
|
||
|
||
for idx, line in enumerate(lines):
|
||
if re.search(r"1\.急诊\s+2\.门诊", line):
|
||
result["入院途径代码"] = first_match(r"^([1-9-])\s+1\.急诊", line)
|
||
if "天" in line and idx in date_line_indexes:
|
||
result["实际住院天数"] = first_match(r"([0-9]+)\s*天\s*$", line)
|
||
|
||
if len(date_line_indexes) >= 2:
|
||
discharge_line = lines[date_line_indexes[1]]
|
||
result["实际住院天数"] = first_match(r"([0-9]+)\s*天\s*$", discharge_line) or result.get("实际住院天数", "")
|
||
if date_line_indexes[1] + 1 < len(lines):
|
||
diagnosis_line = lines[date_line_indexes[1] + 1]
|
||
diagnosis_match = re.match(r"(.+?)\s+([A-Z]\d{2}[\w.]*\d*)\s*$", diagnosis_line)
|
||
if diagnosis_match:
|
||
result["门急诊诊断"] = clean_value(diagnosis_match.group(1))
|
||
result["门急诊诊断编码"] = clean_value(diagnosis_match.group(2))
|
||
|
||
return result
|
||
|
||
|
||
def parse_visit_line(line: str, prefix: str) -> dict[str, str]:
|
||
pattern = (
|
||
r"(?P<year>\d{4})\s*年\s*(?P<month>\d{1,2})\s*月\s*"
|
||
r"(?P<day>\d{1,2})\s*日\s*(?P<hour>\d{1,2})\s*时\s+"
|
||
r"(?P<dept>.+?)\s{2,}(?P<ward>\S+)"
|
||
)
|
||
match = re.search(pattern, line)
|
||
if not match:
|
||
return {}
|
||
return {
|
||
f"{prefix}时间": (
|
||
f"{int(match.group('year')):04d}-{int(match.group('month')):02d}-"
|
||
f"{int(match.group('day')):02d} {int(match.group('hour')):02d}:00"
|
||
),
|
||
f"{prefix}科别": clean_value(match.group("dept")),
|
||
f"{prefix}病房": clean_value(match.group("ward")),
|
||
}
|
||
|
||
|
||
def parse_diagnoses(lines: list[str]) -> dict[str, Any]:
|
||
section = diagnosis_section(lines)
|
||
diagnoses: list[dict[str, str]] = []
|
||
if not section:
|
||
return {"出院诊断": diagnoses, "主要诊断": {}}
|
||
|
||
main_index = next((idx for idx, line in enumerate(section) if "主要诊断" in line), -1)
|
||
if main_index >= 0:
|
||
main_line = section[main_index]
|
||
normalized_main_line = normalize_spaces(main_line)
|
||
main_match = re.search(r"主要诊断\s*(.*?)\s+([A-Z]\d{2}[\w.]*\d*)\s+([0-9-])", normalized_main_line)
|
||
if main_match:
|
||
name = clean_value(main_match.group(1))
|
||
if not name and main_index > 0:
|
||
name = clean_value(section[main_index - 1])
|
||
if main_index + 1 < len(section) and not re.search(r"[A-Z]\d{2}[\w.]*\d*", section[main_index + 1]):
|
||
name = clean_value(name + section[main_index + 1])
|
||
diagnoses.append(
|
||
{
|
||
"诊断类别": "主要诊断",
|
||
"出院诊断": name,
|
||
"疾病编码": clean_value(main_match.group(2)),
|
||
"入院病情": clean_value(main_match.group(3)),
|
||
}
|
||
)
|
||
else:
|
||
no_code_match = re.search(r"主要诊断\s*(.*?)\s+([0-9-])(?:\s+其他诊断)?$", normalized_main_line)
|
||
if no_code_match:
|
||
name = clean_value(no_code_match.group(1))
|
||
if not name:
|
||
nearby_name_parts: list[str] = []
|
||
if main_index > 0:
|
||
previous_line = clean_value(section[main_index - 1])
|
||
if previous_line and "其他诊断" not in previous_line and not re.search(r"[A-Z]\d{2}[\w.]*\d*", previous_line):
|
||
nearby_name_parts.append(previous_line)
|
||
if main_index + 1 < len(section):
|
||
next_line = clean_value(section[main_index + 1])
|
||
if next_line and "其他诊断" not in next_line and not re.search(r"[A-Z]\d{2}[\w.]*\d*", next_line):
|
||
nearby_name_parts.append(next_line)
|
||
name = clean_value("".join(nearby_name_parts))
|
||
if name:
|
||
diagnoses.append(
|
||
{
|
||
"诊断类别": "主要诊断",
|
||
"出院诊断": name,
|
||
"疾病编码": "",
|
||
"入院病情": clean_value(no_code_match.group(2)),
|
||
}
|
||
)
|
||
|
||
other_started = False
|
||
for line in section:
|
||
if "其他诊断" in line:
|
||
other_started = True
|
||
line = re.sub(r"^.*?其他诊断\s*", "", line).strip()
|
||
if not line:
|
||
continue
|
||
elif not other_started:
|
||
continue
|
||
parsed = parse_other_diagnosis_line(line)
|
||
if parsed:
|
||
diagnoses.append(parsed)
|
||
|
||
main = next((item for item in diagnoses if item["诊断类别"] == "主要诊断"), {})
|
||
return {"出院诊断": diagnoses, "主要诊断": main}
|
||
|
||
|
||
def diagnosis_section(lines: list[str]) -> list[str]:
|
||
start = -1
|
||
for idx, line in enumerate(lines):
|
||
if "主要诊断" in line:
|
||
start = max(0, idx - 1)
|
||
break
|
||
if start < 0:
|
||
return []
|
||
|
||
section: list[str] = []
|
||
for line in lines[start:]:
|
||
if "B20" in line or re.match(r"^-+\s+-+$", line) or "1.无 2.有" in line:
|
||
break
|
||
section.append(line)
|
||
return section
|
||
|
||
|
||
def parse_other_diagnosis_line(line: str) -> dict[str, str] | None:
|
||
line = normalize_spaces(line)
|
||
match = re.match(r"^(?:其他诊断\s+)?(.+?)\s+([A-Z]\d{2}[\w.]*\d*)\s+([0-9-])$", line)
|
||
if match:
|
||
return {
|
||
"诊断类别": "其他诊断",
|
||
"出院诊断": clean_value(match.group(1)),
|
||
"疾病编码": clean_value(match.group(2)),
|
||
"入院病情": clean_value(match.group(3)),
|
||
}
|
||
no_code_match = re.match(r"^(?:其他诊断\s+)?(.+?)\s+([0-9-])$", line)
|
||
if no_code_match:
|
||
return {
|
||
"诊断类别": "其他诊断",
|
||
"出院诊断": clean_value(no_code_match.group(1)),
|
||
"疾病编码": "",
|
||
"入院病情": clean_value(no_code_match.group(2)),
|
||
}
|
||
return None
|
||
|
||
|
||
def parse_operations(lines: list[str]) -> list[dict[str, str]]:
|
||
operation_lines: list[str] = []
|
||
after_quality_line = False
|
||
for raw in lines:
|
||
line = raw.replace("\f", "").strip()
|
||
if not line:
|
||
continue
|
||
if "1.甲" in line and "2.乙" in line and "3.丙" in line:
|
||
after_quality_line = True
|
||
continue
|
||
if not after_quality_line and re.match(r"^[A-Z0-9]\d?\.\d", line):
|
||
after_quality_line = True
|
||
if after_quality_line and "1.医嘱离院" in line:
|
||
break
|
||
if after_quality_line:
|
||
operation_lines.append(line)
|
||
|
||
combined: list[str] = []
|
||
pending_prefix = ""
|
||
for line in operation_lines:
|
||
has_date = bool(re.search(r"\d{4}-\d{1,2}-\d{1,2}", line))
|
||
starts_with_code = bool(re.match(r"^[A-Z0-9]\d?\.\d", line))
|
||
if starts_with_code and not has_date:
|
||
pending_prefix = normalize_spaces(line)
|
||
continue
|
||
if has_date:
|
||
if starts_with_code:
|
||
combined.append(line)
|
||
elif pending_prefix:
|
||
combined.append(normalize_spaces(pending_prefix + " " + line))
|
||
pending_prefix = ""
|
||
else:
|
||
combined.append(line)
|
||
elif combined:
|
||
combined[-1] = normalize_spaces(combined[-1] + " " + line)
|
||
|
||
operations: list[dict[str, str]] = []
|
||
for line in combined:
|
||
match = re.match(
|
||
r"^(?:(?P<code>[A-Z0-9.]+[a-zA-Z]*)\s+)?(?:(?P<predate>.*?)\s+)?"
|
||
r"(?P<date>\d{4}-\d{1,2}-\d{1,2})\s+(?P<rest>.+)$",
|
||
line,
|
||
)
|
||
if not match:
|
||
continue
|
||
operation = parse_operation_columns(
|
||
code=clean_value(match.group("code")),
|
||
date=normalize_date(match.group("date")),
|
||
predate=clean_value(match.group("predate") or ""),
|
||
rest=clean_value(match.group("rest")),
|
||
raw_line=clean_value(line),
|
||
)
|
||
operations.append(operation)
|
||
return operations
|
||
|
||
|
||
def parse_operation_columns(code: str, date: str, predate: str, rest: str, raw_line: str) -> dict[str, str]:
|
||
tokens = [clean_operation_token(token) for token in rest.split() if clean_operation_token(token)]
|
||
anesthesia_hint = predate if is_anesthesia_text(predate) else ""
|
||
name_prefix = "" if anesthesia_hint else predate
|
||
|
||
level = ""
|
||
if tokens and is_operation_level_token(tokens[0]):
|
||
raw_level = tokens.pop(0)
|
||
level = normalize_operation_level(raw_level)
|
||
if level == "诊断性操作" and tokens and tokens[-1] == "作":
|
||
level = "诊断性操作"
|
||
tokens.pop()
|
||
|
||
incision_index = next((idx for idx, token in enumerate(tokens) if is_incision_healing_token(token)), -1)
|
||
incision_healing = ""
|
||
if incision_index >= 0:
|
||
before_incision = tokens[:incision_index]
|
||
incision_healing = tokens[incision_index]
|
||
after_incision = tokens[incision_index + 1 :]
|
||
else:
|
||
before_incision = list(tokens)
|
||
after_incision = []
|
||
|
||
if incision_index < 0:
|
||
before_incision, anesthesiologist = split_no_incision_tail(before_incision)
|
||
anesthesia_method = anesthesia_hint
|
||
name_suffix = ""
|
||
else:
|
||
anesthesia_method, anesthesiologist, name_suffix = split_anesthesia_tail(after_incision, anesthesia_hint)
|
||
|
||
procedure_name, surgeon, assistant_1, assistant_2 = split_procedure_and_doctors(before_incision)
|
||
procedure_name = clean_value(" ".join(part for part in [name_prefix, procedure_name, name_suffix] if part))
|
||
|
||
return {
|
||
"手术操作编码": code,
|
||
"手术操作日期": date,
|
||
"手术级别": level,
|
||
"手术操作名称": procedure_name,
|
||
"术者": surgeon,
|
||
"I助": assistant_1,
|
||
"II助": assistant_2,
|
||
"切口愈合等级": incision_healing,
|
||
"麻醉方式": anesthesia_method,
|
||
"麻醉医师": anesthesiologist,
|
||
"原始内容": raw_line,
|
||
}
|
||
|
||
|
||
def clean_operation_token(token: str) -> str:
|
||
token = token.strip().strip("()（）[]【】")
|
||
token = token.replace("Ⅰ", "I").replace("Ⅱ", "II").replace("Ⅲ", "III").replace("Ⅳ", "IV")
|
||
token = token.replace("／", "/")
|
||
return clean_value(token)
|
||
|
||
|
||
def is_operation_level_token(token: str) -> bool:
|
||
return bool(token and (token in {"一级", "二级", "三级", "四级", "诊断性", "诊断性操作", "性操"} or token.endswith("级")))
|
||
|
||
|
||
def normalize_operation_level(token: str) -> str:
|
||
return "诊断性操作" if token in {"性操", "诊断性"} else token
|
||
|
||
|
||
def is_incision_healing_token(token: str) -> bool:
|
||
return bool(re.fullmatch(r"(?:0|I|II|III|IV|V|[一二三四五])/[甲乙丙-]", token))
|
||
|
||
|
||
def is_anesthesia_text(text: str) -> bool:
|
||
return bool(text and re.search(r"麻醉|浸润|静吸|全凭|硬膜|局部", text))
|
||
|
||
|
||
def has_procedure_keyword(token: str) -> bool:
|
||
return bool(re.search(r"术|切除|穿刺|栓塞|成型|松解|活检|操作|置入|修补|吻合", token))
|
||
|
||
|
||
def looks_like_person_name(token: str) -> bool:
|
||
if not re.fullmatch(r"[\u4e00-\u9fa5]{2,4}", token):
|
||
return False
|
||
if has_procedure_keyword(token):
|
||
return False
|
||
if re.search(r"腹腔|经皮|经导管|CT|胆|肝|肺|肠|阑尾|粘连|总管|局部|静吸|全凭|脉麻醉", token):
|
||
return False
|
||
return True
|
||
|
||
|
||
def split_procedure_and_doctors(tokens: list[str]) -> tuple[str, str, str, str]:
|
||
doctor_tokens: list[str] = []
|
||
remaining = list(tokens)
|
||
while remaining and len(doctor_tokens) < 3 and looks_like_person_name(remaining[-1]):
|
||
doctor_tokens.insert(0, remaining.pop())
|
||
procedure_name = clean_value(" ".join(remaining))
|
||
doctor_tokens = doctor_tokens[-3:]
|
||
surgeon = doctor_tokens[0] if len(doctor_tokens) >= 1 else ""
|
||
assistant_1 = doctor_tokens[1] if len(doctor_tokens) >= 2 else ""
|
||
assistant_2 = doctor_tokens[2] if len(doctor_tokens) >= 3 else ""
|
||
return procedure_name, surgeon, assistant_1, assistant_2
|
||
|
||
|
||
def split_anesthesia_tail(tokens: list[str], anesthesia_hint: str = "") -> tuple[str, str, str]:
|
||
clean_tokens = [token for token in (clean_operation_token(item) for item in tokens) if token]
|
||
clean_tokens = [token for token in clean_tokens if not re.fullmatch(r"\d{2,4}", token) and token != "麻醉"]
|
||
doctor_index = next((idx for idx, token in enumerate(clean_tokens) if looks_like_person_name(token)), -1)
|
||
if doctor_index < 0:
|
||
method = clean_value("".join([anesthesia_hint, *clean_tokens]))
|
||
return method, "", ""
|
||
|
||
anesthesiologist = clean_tokens[doctor_index]
|
||
method_parts = clean_tokens[:doctor_index]
|
||
suffix_parts: list[str] = []
|
||
for token in clean_tokens[doctor_index + 1 :]:
|
||
if has_procedure_keyword(token):
|
||
suffix_parts.append(token)
|
||
else:
|
||
method_parts.append(token)
|
||
method = clean_value("".join([anesthesia_hint, *method_parts]))
|
||
name_suffix = clean_value(" ".join(suffix_parts))
|
||
return method, anesthesiologist, name_suffix
|
||
|
||
|
||
def split_no_incision_tail(tokens: list[str]) -> tuple[list[str], str]:
|
||
remaining = [token for token in tokens if token != "麻醉" and not re.fullmatch(r"\d{2,4}", token)]
|
||
if remaining and looks_like_person_name(remaining[-1]):
|
||
return remaining[:-1], remaining[-1]
|
||
return remaining, ""
|
||
|
||
|
||
def normalize_date(value: str) -> str:
|
||
match = re.match(r"(\d{4})-(\d{1,2})-(\d{1,2})", value)
|
||
if not match:
|
||
return value
|
||
return f"{int(match.group(1)):04d}-{int(match.group(2)):02d}-{int(match.group(3)):02d}"
|
||
|
||
|
||
def parse_fees(text: str) -> dict[str, Any]:
|
||
result: dict[str, Any] = {
|
||
"总费用": first_match(r"\n\s*([0-9]+\.[0-9]{2})\s*\(\s*自付金额", text),
|
||
"自付金额": first_match(r"自付金额\s*([0-9]+\.[0-9]{2})", text),
|
||
"费用明细": {},
|
||
}
|
||
for index, name, amount in re.findall(r"\((\d+)\)\s*([^:：()]+?)\s*[:：]\s*([0-9]+(?:\.[0-9]+)?)", text):
|
||
key = f"{int(index):02d}_{clean_value(name)}"
|
||
result["费用明细"][key] = amount
|
||
for name, amount in re.findall(r"(?<!\()\s(麻醉费|手术费|抗菌药物费用|临床物理治疗费)\s*[:：]\s*([0-9]+(?:\.[0-9]+)?)", text):
|
||
result["费用明细"][clean_value(name)] = amount
|
||
return result
|
||
|
||
|
||
def parse_front_page_middle(lines: list[str]) -> dict[str, str]:
|
||
result = {
|
||
"损伤中毒外部原因": "",
|
||
"损伤中毒疾病编码": "",
|
||
"病理诊断": "",
|
||
"病理诊断编码": "",
|
||
"病理号": "",
|
||
"药物过敏代码": "",
|
||
"过敏药物": "",
|
||
"死亡患者尸检代码": "",
|
||
"血型代码": "",
|
||
"Rh代码": "",
|
||
"科主任": "",
|
||
"主任副主任医师": "",
|
||
"主治医师": "",
|
||
"住院医师": "",
|
||
"责任护士": "",
|
||
"进修医师": "",
|
||
"实习医师": "",
|
||
"规培医师": "",
|
||
"编码员": "",
|
||
"病案质量代码": "",
|
||
"质控医师": "",
|
||
"质控护士": "",
|
||
"质控日期": "",
|
||
}
|
||
|
||
pathology_no_idx = next((idx for idx, line in enumerate(lines) if re.match(r"^B\d+", line)), -1)
|
||
if pathology_no_idx >= 0:
|
||
result["病理号"] = clean_value(lines[pathology_no_idx])
|
||
if pathology_no_idx > 0:
|
||
pathology_cols = split_layout_columns(lines[pathology_no_idx - 1])
|
||
if len(pathology_cols) >= 2:
|
||
result["病理诊断"] = clean_value(" ".join(pathology_cols[:-1]))
|
||
result["病理诊断编码"] = clean_value(pathology_cols[-1])
|
||
if pathology_no_idx > 2:
|
||
injury_cols = split_layout_columns(lines[pathology_no_idx - 2])
|
||
if len(injury_cols) >= 2:
|
||
result["损伤中毒外部原因"] = clean_value(injury_cols[0])
|
||
result["损伤中毒疾病编码"] = clean_value(injury_cols[-1])
|
||
|
||
allergy_idx = next((idx for idx, line in enumerate(lines) if "1.无 2.有" in line and "1.是 2.否" in line), -1)
|
||
if allergy_idx >= 0:
|
||
match = re.search(r"^\s*([1-9-])\s+1\.无 2\.有\s+(.*?)\s+([1-9-])\s+1\.是 2\.否", lines[allergy_idx])
|
||
if match:
|
||
result["药物过敏代码"] = clean_value(match.group(1))
|
||
result["过敏药物"] = clean_value(match.group(2))
|
||
result["死亡患者尸检代码"] = clean_value(match.group(3))
|
||
|
||
blood_idx = next((idx for idx, line in enumerate(lines) if "1.A" in line and "1.阴" in line), -1)
|
||
if blood_idx >= 0:
|
||
result["血型代码"] = first_match(r"^\s*([1-9-])\s+1\.A", lines[blood_idx])
|
||
result["Rh代码"] = first_match(r"6\.未查\s+([1-9-])\s+1\.阴", lines[blood_idx])
|
||
doctor_rows = [split_layout_columns_keep_positions(lines[i]) for i in range(blood_idx + 1, min(blood_idx + 3, len(lines)))]
|
||
if doctor_rows:
|
||
for key, value in zip(["科主任", "主任副主任医师", "主治医师", "住院医师"], doctor_rows[0]):
|
||
result[key] = clean_value(value)
|
||
if len(doctor_rows) >= 2:
|
||
for key, value in zip(["责任护士", "进修医师", "实习医师", "规培医师", "编码员"], doctor_rows[1]):
|
||
result[key] = clean_value(value)
|
||
|
||
quality_idx = next((idx for idx, line in enumerate(lines) if "1.甲" in line and "2.乙" in line and "3.丙" in line), -1)
|
||
if quality_idx >= 0:
|
||
line = lines[quality_idx]
|
||
result["病案质量代码"] = first_match(r"^\s*([1-9-])\s+1\.甲", line)
|
||
date_match = re.search(r"(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*日", line)
|
||
if date_match:
|
||
result["质控日期"] = f"{int(date_match.group(1)):04d}-{int(date_match.group(2)):02d}-{int(date_match.group(3)):02d}"
|
||
line = line[: date_match.start()]
|
||
line = re.sub(r"^\s*[1-9-]\s+1\.甲 2\.乙 3\.丙\s*", "", line)
|
||
cols = split_layout_columns(line)
|
||
if len(cols) >= 1:
|
||
result["质控医师"] = clean_value(cols[-2] if len(cols) >= 2 else cols[0])
|
||
if len(cols) >= 2:
|
||
result["质控护士"] = clean_value(cols[-1])
|
||
|
||
return result
|
||
|
||
|
||
def split_layout_columns(line: str) -> list[str]:
|
||
return [clean_value(part) for part in re.split(r"\s{2,}", line.strip()) if clean_value(part)]
|
||
|
||
|
||
def split_layout_columns_keep_positions(line: str) -> list[str]:
|
||
return [clean_value(normalize_spaces(part)) for part in re.split(r"\s{2,}", line.strip()) if part.strip()]
|
||
|
||
|
||
def parse_discharge_followup(lines: list[str]) -> dict[str, str]:
|
||
result = {
|
||
"离院方式代码": "",
|
||
"拟接收医疗机构名称": "",
|
||
"出院31天内再住院计划代码": "",
|
||
"再住院计划目的": "",
|
||
"入院前昏迷天数": "",
|
||
"入院前昏迷小时": "",
|
||
"入院前昏迷分钟": "",
|
||
"入院后昏迷天数": "",
|
||
"入院后昏迷小时": "",
|
||
"入院后昏迷分钟": "",
|
||
}
|
||
for line in lines:
|
||
if "1.医嘱离院" in line:
|
||
result["离院方式代码"] = first_match(r"^\s*([1-9-])\s+1\.医嘱离院", line)
|
||
result["拟接收医疗机构名称"] = first_match(r"拟接收医疗机构名称:\s*(.*?)\s*$", line)
|
||
elif "是否有出院31天内再住院计划" in line:
|
||
result["出院31天内再住院计划代码"] = first_match(r"是否有出院31天内再住院计划\s*([1-9-])", line)
|
||
result["再住院计划目的"] = first_match(r"目的:\s*(.*?)\s*$", line)
|
||
elif "入院前" in line and "入院后" in line and "分钟" in line:
|
||
match = re.search(
|
||
r"入院前\s*([-\d]+)\s*天\s*([-\d]+)\s*小时\s*([-\d]+)\s*分钟\s*"
|
||
r"入院后\s*([-\d]+)\s*天\s*([-\d]+)\s*小时\s*([-\d]+)\s*分钟",
|
||
line,
|
||
)
|
||
if match:
|
||
result["入院前昏迷天数"] = clean_int_value(match.group(1))
|
||
result["入院前昏迷小时"] = clean_int_value(match.group(2))
|
||
result["入院前昏迷分钟"] = clean_int_value(match.group(3))
|
||
result["入院后昏迷天数"] = clean_int_value(match.group(4))
|
||
result["入院后昏迷小时"] = clean_int_value(match.group(5))
|
||
result["入院后昏迷分钟"] = clean_int_value(match.group(6))
|
||
return result
|
||
|
||
|
||
def build_quality_warnings(record: dict[str, Any], text: str) -> list[str]:
|
||
warnings: list[str] = []
|
||
required = ["病案号", "姓名", "性别", "入院时间", "出院时间", "主要诊断"]
|
||
for key in required:
|
||
value = record.get(key)
|
||
if isinstance(value, dict):
|
||
if not value:
|
||
warnings.append(f"缺少{key}")
|
||
elif not value:
|
||
warnings.append(f"缺少{key}")
|
||
if len(text.strip()) < 200:
|
||
warnings.append("提取文本过短，可能是扫描件或加密PDF")
|
||
return warnings
|
||
|
||
|
||
def validate_record(record: dict[str, Any], department_rules_loaded: bool = False) -> list[str]:
|
||
notes: list[str] = []
|
||
if not clean_value(str(record.get("住院号", ""))):
|
||
notes.append("患者号/住院号为空")
|
||
if not re.fullmatch(r"\d{10}", str(record.get("病案号", ""))):
|
||
notes.append("病案号不是10位数字")
|
||
if record.get("性别") and record["性别"] not in {"男", "女"}:
|
||
notes.append("性别不是男/女")
|
||
if record.get("身份证号") and not re.fullmatch(r"\d{17}[\dX]", record["身份证号"]):
|
||
notes.append("身份证号格式异常")
|
||
if record.get("出生日期") and not re.fullmatch(r"\d{4}-\d{2}-\d{2}", record["出生日期"]):
|
||
notes.append("出生日期格式异常")
|
||
if record.get("入院时间") and record.get("出院时间") and record["入院时间"] > record["出院时间"]:
|
||
notes.append("出院时间早于入院时间")
|
||
if record.get("实际住院天数") and not re.fullmatch(r"\d+", str(record["实际住院天数"])):
|
||
notes.append("实际住院天数不是整数")
|
||
|
||
main = record.get("主要诊断") if isinstance(record.get("主要诊断"), dict) else {}
|
||
if main and not re.match(r"^[A-Z]\d{2}", main.get("疾病编码", "")):
|
||
notes.append("主要诊断编码格式异常")
|
||
other_diagnoses = [diagnosis for diagnosis in record.get("出院诊断", []) if diagnosis.get("诊断类别") == "其他诊断"]
|
||
for index, diagnosis in enumerate(other_diagnoses, start=1):
|
||
if not re.match(r"^[A-Z]\d{2}", diagnosis.get("疾病编码", "")):
|
||
notes.append(f"其他诊断{index}编码格式异常")
|
||
suspicious_phrases = ["1.急诊", "2.门诊", "其他医疗机构转入", "1.医嘱离院"]
|
||
for key in ["联系人姓名", "联系人关系", "联系人地址", "联系人电话", "现住址", "户口地址", "工作单位及地址"]:
|
||
value = str(record.get(key, ""))
|
||
if any(phrase in value for phrase in suspicious_phrases):
|
||
notes.append(f"{key}疑似串入版式选项文本")
|
||
for index, operation in enumerate(record.get("手术操作", []), start=1):
|
||
if not operation.get("手术操作日期") or not operation.get("手术操作名称"):
|
||
notes.append(f"手术操作{index}缺少日期或名称")
|
||
if not operation.get("手术操作编码"):
|
||
notes.append(f"手术操作{index}缺少手术及操作编码")
|
||
|
||
for key in ["总费用", "自付金额"]:
|
||
if record.get(key) and not re.fullmatch(r"\d+(\.\d{1,2})?", str(record[key])):
|
||
notes.append(f"{key}不是金额格式")
|
||
if department_rules_loaded and (record.get("出院科别") or record.get("入院科别")) and not record.get("大科室"):
|
||
notes.append("科别未匹配到大科室分类")
|
||
return notes
|
||
|
||
|
||
def apply_review_fields(record: dict[str, Any], validation_notes: list[str]) -> None:
|
||
auto_corrections = [
|
||
note for note in record.get("自动修正", [])
|
||
if not ("病案号由PDF值" in note and "按文件名补齐" in note)
|
||
]
|
||
quality_notes = record.get("质控提示", [])
|
||
review_notes = list(dict.fromkeys([*quality_notes, *validation_notes, *record.get("复核备注", [])]))
|
||
record["自动修正"] = auto_corrections
|
||
record["复核备注"] = review_notes
|
||
record["人工修正"] = False
|
||
record["manual_corrected"] = False
|
||
if review_notes:
|
||
record["复核状态"] = "needs_review"
|
||
elif auto_corrections:
|
||
record["复核状态"] = "auto_corrected"
|
||
else:
|
||
record["复核状态"] = "auto_pass"
|
||
|
||
|
||
def parse_record(
|
||
pdf_path: Path,
|
||
department_rules: dict[str, dict[str, str]] | None = None,
|
||
args: argparse.Namespace | None = None,
|
||
) -> tuple[dict[str, Any], str]:
|
||
if args is None:
|
||
args = build_parser().parse_args([])
|
||
text, text_method, mineru_markdown_dir = extract_record_text(pdf_path, args)
|
||
lines = meaningful_lines(text)
|
||
|
||
diagnoses = parse_diagnoses(lines)
|
||
fees = parse_fees(text)
|
||
header = parse_header(text, lines)
|
||
normalized_no, original_no, corrections, no_notes = normalize_medical_record_no(header.get("病案号", ""), pdf_path)
|
||
header["病案号"] = normalized_no
|
||
header["首页病案号"] = normalize_digits(original_no or normalized_no, 10)
|
||
inpatient_no, inpatient_corrections, inpatient_notes = build_inpatient_no(
|
||
header.get("住院次数", ""),
|
||
header.get("首页病案号", ""),
|
||
header.get("病案号", ""),
|
||
pdf_path,
|
||
)
|
||
record: dict[str, Any] = {
|
||
"源文件": pdf_path.name,
|
||
"住院号": inpatient_no,
|
||
"解析时间": datetime.now().isoformat(timespec="seconds"),
|
||
"解析器版本": "patient-front-page-local-v1",
|
||
"文本抽取方式": text_method,
|
||
"Mineru Markdown目录": mineru_markdown_dir,
|
||
"自动修正": [*corrections, *inpatient_corrections],
|
||
"复核备注": [*no_notes, *inpatient_notes],
|
||
}
|
||
record.update(header)
|
||
record.update(parse_newborn_info(lines))
|
||
record.update(parse_basic_lines(lines))
|
||
record.update(parse_admission_discharge(lines))
|
||
department_rules = department_rules or {}
|
||
major_department, standard_department = classify_major_department(record, department_rules)
|
||
record["大科室"] = major_department
|
||
record["标准子科室"] = standard_department
|
||
record.update(diagnoses)
|
||
record["手术操作"] = parse_operations(lines)
|
||
record.update(parse_front_page_middle(lines))
|
||
record.update(parse_discharge_followup(lines))
|
||
record.update(fees)
|
||
record["原始文本"] = text
|
||
|
||
main = diagnoses.get("主要诊断") or {}
|
||
if main:
|
||
record["主要诊断名称"] = main.get("出院诊断", "")
|
||
record["主要诊断编码"] = main.get("疾病编码", "")
|
||
record["主要诊断入院病情"] = main.get("入院病情", "")
|
||
|
||
warnings = build_quality_warnings(record, text)
|
||
validation_notes = validate_record(record, department_rules_loaded=bool(department_rules))
|
||
record["质控状态"] = "需复核" if warnings else "通过"
|
||
record["质控提示"] = warnings
|
||
apply_review_fields(record, validation_notes)
|
||
return record, text
|
||
|
||
|
||
def flatten_for_csv(record: dict[str, Any]) -> dict[str, str]:
|
||
main = record.get("主要诊断") if isinstance(record.get("主要诊断"), dict) else {}
|
||
return {
|
||
"住院号": record.get("住院号", ""),
|
||
"源文件": record.get("源文件", ""),
|
||
"病案号": record.get("病案号", ""),
|
||
"首页病案号": record.get("首页病案号", ""),
|
||
"姓名": record.get("姓名", ""),
|
||
"性别": record.get("性别", ""),
|
||
"出生日期": record.get("出生日期", ""),
|
||
"年龄": record.get("年龄", ""),
|
||
"身份证号": record.get("身份证号", ""),
|
||
"新生儿年龄（月）": record.get("新生儿年龄（月）", ""),
|
||
"新生儿出生体重（克）": record.get("新生儿出生体重（克）", ""),
|
||
"新生儿入院体重（克）": record.get("新生儿入院体重（克）", ""),
|
||
"医疗机构": record.get("医疗机构", ""),
|
||
"组织机构代码": record.get("组织机构代码", ""),
|
||
"医疗付费方式": record.get("医疗付费方式", ""),
|
||
"住院次数": record.get("住院次数", ""),
|
||
"入院时间": record.get("入院时间", ""),
|
||
"入院科别": record.get("入院科别", ""),
|
||
"入院病房": record.get("入院病房", ""),
|
||
"转科科别": record.get("转科科别", ""),
|
||
"转科时间": record.get("转科时间", ""),
|
||
"出院时间": record.get("出院时间", ""),
|
||
"出院科别": record.get("出院科别", ""),
|
||
"出院病房": record.get("出院病房", ""),
|
||
"大科室": record.get("大科室", ""),
|
||
"实际住院天数": record.get("实际住院天数", ""),
|
||
"门急诊诊断": record.get("门急诊诊断", ""),
|
||
"门急诊诊断编码": record.get("门急诊诊断编码", ""),
|
||
"主要诊断": main.get("出院诊断", ""),
|
||
"主要诊断编码": main.get("疾病编码", ""),
|
||
"主要诊断入院病情": main.get("入院病情", ""),
|
||
"出院诊断": json.dumps(record.get("出院诊断", []), ensure_ascii=False),
|
||
"手术操作": json.dumps(record.get("手术操作", []), ensure_ascii=False),
|
||
"病理诊断": record.get("病理诊断", ""),
|
||
"病理诊断编码": record.get("病理诊断编码", ""),
|
||
"病理号": record.get("病理号", ""),
|
||
"药物过敏代码": record.get("药物过敏代码", ""),
|
||
"过敏药物": record.get("过敏药物", ""),
|
||
"死亡患者尸检代码": record.get("死亡患者尸检代码", ""),
|
||
"血型代码": record.get("血型代码", ""),
|
||
"Rh代码": record.get("Rh代码", ""),
|
||
"科主任": record.get("科主任", ""),
|
||
"主任副主任医师": record.get("主任副主任医师", ""),
|
||
"主治医师": record.get("主治医师", ""),
|
||
"住院医师": record.get("住院医师", ""),
|
||
"责任护士": record.get("责任护士", ""),
|
||
"进修医师": record.get("进修医师", ""),
|
||
"实习医师": record.get("实习医师", ""),
|
||
"规培医师": record.get("规培医师", ""),
|
||
"编码员": record.get("编码员", ""),
|
||
"病案质量代码": record.get("病案质量代码", ""),
|
||
"质控医师": record.get("质控医师", ""),
|
||
"质控护士": record.get("质控护士", ""),
|
||
"质控日期": record.get("质控日期", ""),
|
||
"离院方式代码": record.get("离院方式代码", ""),
|
||
"出院31天内再住院计划代码": record.get("出院31天内再住院计划代码", ""),
|
||
"总费用": record.get("总费用", ""),
|
||
"自付金额": record.get("自付金额", ""),
|
||
"质控状态": record.get("质控状态", ""),
|
||
"质控提示": "；".join(record.get("质控提示", [])),
|
||
"复核状态": record.get("复核状态", ""),
|
||
"复核备注": "；".join(record.get("复核备注", [])),
|
||
"文本抽取方式": record.get("文本抽取方式", ""),
|
||
"自动修正": "；".join(record.get("自动修正", [])),
|
||
"人工修正": str(record.get("人工修正", False)),
|
||
}
|
||
|
||
|
||
def write_outputs(records: list[dict[str, Any]], output_dir: Path, save_text: bool) -> None:
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
structured_dir = output_dir / "01_结构化结果"
|
||
per_record_dir = output_dir / "02_单份JSON"
|
||
text_dir = output_dir / "03_提取文本"
|
||
review_dir = output_dir / "04_复核与人工校正"
|
||
structured_dir.mkdir(exist_ok=True)
|
||
per_record_dir.mkdir(exist_ok=True)
|
||
review_dir.mkdir(exist_ok=True)
|
||
if save_text:
|
||
text_dir.mkdir(exist_ok=True)
|
||
|
||
jsonl_path = structured_dir / "患者首页_结构化结果.jsonl"
|
||
csv_path = structured_dir / "患者首页_结构化结果.csv"
|
||
review_path = review_dir / "患者首页_复核清单.csv"
|
||
|
||
with jsonl_path.open("w", encoding="utf-8") as fp:
|
||
for record in records:
|
||
fp.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||
|
||
with csv_path.open("w", encoding="utf-8-sig", newline="") as fp:
|
||
writer = csv.DictWriter(fp, fieldnames=CSV_COLUMNS)
|
||
writer.writeheader()
|
||
for record in records:
|
||
writer.writerow(flatten_for_csv(record))
|
||
|
||
with review_path.open("w", encoding="utf-8-sig", newline="") as fp:
|
||
writer = csv.DictWriter(fp, fieldnames=["住院号", "源文件", "病案号", "首页病案号", "姓名", "复核状态", "复核备注", "自动修正", "人工修正"])
|
||
writer.writeheader()
|
||
for record in records:
|
||
if record.get("复核状态") != "auto_pass":
|
||
writer.writerow(
|
||
{
|
||
"住院号": record.get("住院号", ""),
|
||
"源文件": record.get("源文件", ""),
|
||
"病案号": record.get("病案号", ""),
|
||
"首页病案号": record.get("首页病案号", ""),
|
||
"姓名": record.get("姓名", ""),
|
||
"复核状态": record.get("复核状态", ""),
|
||
"复核备注": "；".join(record.get("复核备注", [])),
|
||
"自动修正": "；".join(record.get("自动修正", [])),
|
||
"人工修正": str(record.get("人工修正", False)),
|
||
}
|
||
)
|
||
|
||
for record in records:
|
||
stem = Path(record["源文件"]).stem
|
||
json_path = per_record_dir / f"{stem}.json"
|
||
with json_path.open("w", encoding="utf-8") as fp:
|
||
json.dump(record, fp, ensure_ascii=False, indent=2)
|
||
if save_text:
|
||
(text_dir / f"{stem}.txt").write_text(record.get("原始文本", ""), encoding="utf-8")
|
||
|
||
|
||
def quote_pg_identifier(identifier: str) -> str:
|
||
if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", identifier):
|
||
raise ValueError(f"非法 PostgreSQL 表名：{identifier}")
|
||
return '"' + identifier.replace('"', '""') + '"'
|
||
|
||
|
||
def pg_literal(value: str) -> str:
|
||
return "'" + value.replace("'", "''") + "'"
|
||
|
||
|
||
def int_or_empty(value: Any) -> str:
|
||
value = clean_value(str(value)) if value is not None else ""
|
||
return value if re.fullmatch(r"\d+", value) else ""
|
||
|
||
|
||
def decimal_or_empty(value: Any) -> str:
|
||
value = clean_value(str(value)) if value is not None else ""
|
||
return value if re.fullmatch(r"\d+(\.\d{1,2})?", value) else ""
|
||
|
||
|
||
def pg_json(value: Any) -> str:
|
||
return json.dumps(value if value is not None else [], ensure_ascii=False)
|
||
|
||
|
||
def record_to_pg_row(record: dict[str, Any]) -> dict[str, str]:
|
||
main = record.get("主要诊断") if isinstance(record.get("主要诊断"), dict) else {}
|
||
return {
|
||
"source_file": record.get("源文件", ""),
|
||
"inpatient_no": record.get("住院号", ""),
|
||
"medical_record_no": record.get("病案号", ""),
|
||
"front_page_medical_record_no": record.get("首页病案号", ""),
|
||
"patient_name": record.get("姓名", ""),
|
||
"gender": record.get("性别", ""),
|
||
"birth_date": record.get("出生日期", ""),
|
||
"age": record.get("年龄", ""),
|
||
"nationality": record.get("国籍", ""),
|
||
"id_card_no": record.get("身份证号", ""),
|
||
"neonatal_age_months": int_or_empty(record.get("新生儿年龄（月）", "")),
|
||
"newborn_birth_weight_g": int_or_empty(record.get("新生儿出生体重（克）", "")),
|
||
"newborn_admission_weight_g": int_or_empty(record.get("新生儿入院体重（克）", "")),
|
||
"hospital_name": record.get("医疗机构", ""),
|
||
"organization_code": record.get("组织机构代码", ""),
|
||
"payment_method": record.get("医疗付费方式", ""),
|
||
"health_card_no": record.get("健康卡号", ""),
|
||
"admission_count": int_or_empty(record.get("住院次数", "")),
|
||
"birthplace": record.get("出生地", ""),
|
||
"native_place": record.get("籍贯", ""),
|
||
"ethnicity": record.get("民族", ""),
|
||
"occupation": record.get("职业", ""),
|
||
"marital_status_code": record.get("婚姻代码", ""),
|
||
"current_address": record.get("现住址", ""),
|
||
"current_address_phone": record.get("现住址电话", ""),
|
||
"current_address_postcode": record.get("现住址邮编", ""),
|
||
"household_address": record.get("户口地址", ""),
|
||
"household_postcode": record.get("户口地址邮编", ""),
|
||
"employer_address": record.get("工作单位及地址", ""),
|
||
"employer_phone": record.get("单位电话", ""),
|
||
"employer_postcode": record.get("单位邮编", ""),
|
||
"contact_name": record.get("联系人姓名", ""),
|
||
"contact_relationship": record.get("联系人关系", ""),
|
||
"contact_address": record.get("联系人地址", ""),
|
||
"contact_phone": record.get("联系人电话", ""),
|
||
"admission_path_code": record.get("入院途径代码", ""),
|
||
"admission_time": record.get("入院时间", ""),
|
||
"admission_dept": record.get("入院科别", ""),
|
||
"admission_ward": record.get("入院病房", ""),
|
||
"transfer_dept": record.get("转科科别", ""),
|
||
"transfer_time": record.get("转科时间", ""),
|
||
"discharge_time": record.get("出院时间", ""),
|
||
"discharge_dept": record.get("出院科别", ""),
|
||
"discharge_ward": record.get("出院病房", ""),
|
||
"major_department": record.get("大科室", ""),
|
||
"hospital_days": int_or_empty(record.get("实际住院天数", "")),
|
||
"outpatient_diagnosis": record.get("门急诊诊断", ""),
|
||
"outpatient_diagnosis_code": record.get("门急诊诊断编码", ""),
|
||
"primary_diagnosis": main.get("出院诊断", ""),
|
||
"primary_diagnosis_code": main.get("疾病编码", ""),
|
||
"primary_admission_condition": main.get("入院病情", ""),
|
||
"discharge_diagnoses": pg_json(record.get("出院诊断", [])),
|
||
"operations": pg_json(record.get("手术操作", [])),
|
||
"injury_poisoning_external_cause": record.get("损伤中毒外部原因", ""),
|
||
"injury_poisoning_code": record.get("损伤中毒疾病编码", ""),
|
||
"pathology_diagnosis": record.get("病理诊断", ""),
|
||
"pathology_diagnosis_code": record.get("病理诊断编码", ""),
|
||
"pathology_no": record.get("病理号", ""),
|
||
"drug_allergy_code": record.get("药物过敏代码", ""),
|
||
"allergy_drug": record.get("过敏药物", ""),
|
||
"autopsy_code": record.get("死亡患者尸检代码", ""),
|
||
"blood_type_code": record.get("血型代码", ""),
|
||
"rh_code": record.get("Rh代码", ""),
|
||
"department_director": record.get("科主任", ""),
|
||
"chief_physician": record.get("主任副主任医师", ""),
|
||
"attending_physician": record.get("主治医师", ""),
|
||
"resident_physician": record.get("住院医师", ""),
|
||
"responsible_nurse": record.get("责任护士", ""),
|
||
"refresher_physician": record.get("进修医师", ""),
|
||
"intern_physician": record.get("实习医师", ""),
|
||
"standardized_resident_physician": record.get("规培医师", ""),
|
||
"coder": record.get("编码员", ""),
|
||
"record_quality_code": record.get("病案质量代码", ""),
|
||
"quality_control_physician": record.get("质控医师", ""),
|
||
"quality_control_nurse": record.get("质控护士", ""),
|
||
"quality_control_date": record.get("质控日期", ""),
|
||
"discharge_disposition_code": record.get("离院方式代码", ""),
|
||
"receiving_org_name": record.get("拟接收医疗机构名称", ""),
|
||
"readmission_plan_code": record.get("出院31天内再住院计划代码", ""),
|
||
"readmission_plan_purpose": record.get("再住院计划目的", ""),
|
||
"coma_before_days": int_or_empty(record.get("入院前昏迷天数", "")),
|
||
"coma_before_hours": int_or_empty(record.get("入院前昏迷小时", "")),
|
||
"coma_before_minutes": int_or_empty(record.get("入院前昏迷分钟", "")),
|
||
"coma_after_days": int_or_empty(record.get("入院后昏迷天数", "")),
|
||
"coma_after_hours": int_or_empty(record.get("入院后昏迷小时", "")),
|
||
"coma_after_minutes": int_or_empty(record.get("入院后昏迷分钟", "")),
|
||
"total_cost": decimal_or_empty(record.get("总费用", "")),
|
||
"self_pay_amount": decimal_or_empty(record.get("自付金额", "")),
|
||
"fee_details": pg_json(record.get("费用明细", {})),
|
||
"quality_status": record.get("质控状态", ""),
|
||
"quality_notes": pg_json(record.get("质控提示", [])),
|
||
"review_status": record.get("复核状态", "pending"),
|
||
"review_notes": pg_json(record.get("复核备注", [])),
|
||
"manual_corrected": "true" if record.get("manual_corrected") or record.get("人工修正") else "false",
|
||
"auto_corrections": pg_json(record.get("自动修正", [])),
|
||
"text_extraction_method": record.get("文本抽取方式", ""),
|
||
"mineru_markdown_dir": record.get("Mineru Markdown目录", ""),
|
||
"raw_text": record.get("原始文本", ""),
|
||
}
|
||
|
||
|
||
def pg_copy_cast(column_name: str, column_type: str) -> str:
|
||
if column_name == "inpatient_no":
|
||
return "NULLIF(BTRIM(inpatient_no), '')"
|
||
if column_type in {"INTEGER"}:
|
||
return f"NULLIF({column_name}, '')::{column_type}"
|
||
if column_type.startswith("NUMERIC"):
|
||
return f"NULLIF({column_name}, '')::{column_type}"
|
||
if column_type in {"DATE", "TIMESTAMP"}:
|
||
return f"NULLIF({column_name}, '')::{column_type}"
|
||
if column_type.startswith("JSONB"):
|
||
return f"COALESCE(NULLIF({column_name}, ''), 'null')::jsonb"
|
||
if column_type.startswith("BOOLEAN"):
|
||
return f"COALESCE(NULLIF({column_name}, '')::boolean, false)"
|
||
return f"NULLIF({column_name}, '')"
|
||
|
||
|
||
def write_postgres(records: list[dict[str, Any]], args: argparse.Namespace) -> None:
|
||
if not records:
|
||
return
|
||
if shutil.which("psql") is None:
|
||
raise RuntimeError("未找到 psql。请先安装 PostgreSQL 客户端，或取消 --write-postgres。")
|
||
|
||
table_name = quote_pg_identifier(args.pg_table)
|
||
list_table_name = quote_pg_identifier("Patient_Lists")
|
||
trigger_function_name = quote_pg_identifier(f"{args.pg_table}_sync_patient_lists_trigger_fn")
|
||
trigger_name = quote_pg_identifier(f"trg_{args.pg_table}_sync_patient_lists")
|
||
dedupe_trigger_function_name = quote_pg_identifier(f"{args.pg_table}_dedupe_inpatient_no_trigger_fn")
|
||
dedupe_trigger_name = quote_pg_identifier(f"trg_{args.pg_table}_dedupe_inpatient_no")
|
||
source_file_constraint = quote_pg_identifier(f"{args.pg_table}_source_file_key")
|
||
inpatient_no_constraint = quote_pg_identifier(f"{args.pg_table}_inpatient_no_key")
|
||
inpatient_no_check_constraints = [
|
||
quote_pg_identifier(f"ck_{args.pg_table}_inpatient_no_format"),
|
||
quote_pg_identifier(f"ck_{args.pg_table.lower()}_inpatient_no_format"),
|
||
quote_pg_identifier(f"ck_{args.pg_table}_inpatient_no_required"),
|
||
]
|
||
inpatient_no_required_constraint = quote_pg_identifier(f"ck_{args.pg_table}_inpatient_no_required")
|
||
list_inpatient_no_check_constraints = [
|
||
quote_pg_identifier("ck_Patient_Lists_inpatient_no_format"),
|
||
quote_pg_identifier("ck_patient_lists_inpatient_no_format"),
|
||
quote_pg_identifier("ck_patient_lists_inpatient_no_required"),
|
||
]
|
||
list_inpatient_no_required_constraint = quote_pg_identifier("ck_patient_lists_inpatient_no_required")
|
||
connection_env = os.environ.copy()
|
||
connection_env.update(
|
||
{
|
||
"PGHOST": args.pg_host or os.environ.get("PGHOST", ""),
|
||
"PGPORT": str(args.pg_port or os.environ.get("PGPORT", "")),
|
||
"PGDATABASE": args.pg_database or os.environ.get("PGDATABASE", ""),
|
||
"PGUSER": args.pg_user or os.environ.get("PGUSER", ""),
|
||
"PGPASSWORD": args.pg_password or os.environ.get("PGPASSWORD", ""),
|
||
}
|
||
)
|
||
missing = [name for name in ["PGHOST", "PGPORT", "PGDATABASE", "PGUSER", "PGPASSWORD"] if not connection_env.get(name)]
|
||
if missing:
|
||
raise RuntimeError("缺少 PostgreSQL 连接配置：" + "、".join(missing))
|
||
|
||
with tempfile.TemporaryDirectory(prefix="patient_front_pages_") as tmpdir:
|
||
csv_path = Path(tmpdir) / "records.csv"
|
||
sql_path = Path(tmpdir) / "import.sql"
|
||
with csv_path.open("w", encoding="utf-8", newline="") as fp:
|
||
fieldnames = [name for name, _type in PG_COLUMNS]
|
||
writer = csv.DictWriter(fp, fieldnames=fieldnames)
|
||
writer.writeheader()
|
||
for record in records:
|
||
writer.writerow(record_to_pg_row(record))
|
||
|
||
escaped_csv = str(csv_path).replace("'", "''")
|
||
column_defs = ",\n ".join(
|
||
["id BIGSERIAL PRIMARY KEY", *[f"{name} {column_type}" for name, column_type in PG_COLUMNS], "UNIQUE (inpatient_no)"]
|
||
)
|
||
alter_add_columns = "\n".join(
|
||
[f"ALTER TABLE {table_name} ADD COLUMN IF NOT EXISTS {name} {column_type};" for name, column_type in PG_COLUMNS]
|
||
)
|
||
table_comments = "\n".join(
|
||
[
|
||
f"COMMENT ON COLUMN {table_name}.{quote_pg_identifier(column_name)} IS {pg_literal(comment)};"
|
||
for column_name, comment in PG_COLUMN_COMMENTS.items()
|
||
]
|
||
)
|
||
import_column_defs = ",\n ".join([f"{name} TEXT" for name, _type in PG_COLUMNS])
|
||
import_columns = ", ".join([name for name, _type in PG_COLUMNS])
|
||
select_values = ",\n ".join([pg_copy_cast(name, column_type) for name, column_type in PG_COLUMNS])
|
||
update_values = ",\n ".join([f"{name} = EXCLUDED.{name}" for name, _type in PG_COLUMNS if name != "inpatient_no"])
|
||
dedupe_update_values = ",\n ".join([f"{name} = NEW.{name}" for name, _type in PG_COLUMNS if name != "inpatient_no"])
|
||
drop_inpatient_no_checks = "\n".join(
|
||
[f"ALTER TABLE {table_name} DROP CONSTRAINT IF EXISTS {constraint};" for constraint in inpatient_no_check_constraints]
|
||
)
|
||
drop_list_inpatient_no_checks = "\n".join(
|
||
[
|
||
f"ALTER TABLE {list_table_name} DROP CONSTRAINT IF EXISTS {constraint};"
|
||
for constraint in list_inpatient_no_check_constraints
|
||
]
|
||
)
|
||
sql_path.write_text(
|
||
f"""
|
||
\\set ON_ERROR_STOP on
|
||
CREATE TABLE IF NOT EXISTS {table_name} (
|
||
{column_defs}
|
||
);
|
||
|
||
ALTER TABLE {table_name} DROP COLUMN IF EXISTS payload;
|
||
ALTER TABLE {table_name} DROP COLUMN IF EXISTS parsed_at;
|
||
ALTER TABLE {table_name} DROP COLUMN IF EXISTS created_at;
|
||
ALTER TABLE {table_name} DROP COLUMN IF EXISTS updated_at;
|
||
DO $$
|
||
BEGIN
|
||
IF EXISTS (
|
||
SELECT 1 FROM information_schema.columns
|
||
WHERE table_schema = 'public' AND table_name = {args.pg_table!r} AND column_name = 'original_medical_record_no'
|
||
) AND NOT EXISTS (
|
||
SELECT 1 FROM information_schema.columns
|
||
WHERE table_schema = 'public' AND table_name = {args.pg_table!r} AND column_name = 'front_page_medical_record_no'
|
||
) THEN
|
||
ALTER TABLE {table_name} RENAME COLUMN original_medical_record_no TO front_page_medical_record_no;
|
||
END IF;
|
||
|
||
IF EXISTS (
|
||
SELECT 1 FROM information_schema.columns
|
||
WHERE table_schema = 'public' AND table_name = {args.pg_table!r} AND column_name = 'diagnoses'
|
||
) AND NOT EXISTS (
|
||
SELECT 1 FROM information_schema.columns
|
||
WHERE table_schema = 'public' AND table_name = {args.pg_table!r} AND column_name = 'discharge_diagnoses'
|
||
) THEN
|
||
ALTER TABLE {table_name} RENAME COLUMN diagnoses TO discharge_diagnoses;
|
||
END IF;
|
||
END $$;
|
||
ALTER TABLE {table_name} DROP COLUMN IF EXISTS other_diagnoses;
|
||
{alter_add_columns}
|
||
UPDATE {table_name}
|
||
SET front_page_medical_record_no = RIGHT(LPAD(regexp_replace(front_page_medical_record_no, '\\D', '', 'g'), 10, '0'), 10)
|
||
WHERE front_page_medical_record_no IS NOT NULL
|
||
AND front_page_medical_record_no <> ''
|
||
AND front_page_medical_record_no !~ '^\\d{{10}}$'
|
||
AND regexp_replace(front_page_medical_record_no, '\\D', '', 'g') <> '';
|
||
UPDATE {table_name}
|
||
SET inpatient_no =
|
||
'ZY'
|
||
|| COALESCE(
|
||
LPAD(admission_count::text, 2, '0'),
|
||
substring(source_file from '^ZY([0-9]{{2}})[0-9]{{10}}')
|
||
)
|
||
|| RIGHT(
|
||
LPAD(
|
||
COALESCE(
|
||
NULLIF(regexp_replace(COALESCE(front_page_medical_record_no, ''), '\\D', '', 'g'), ''),
|
||
NULLIF(regexp_replace(COALESCE(medical_record_no, ''), '\\D', '', 'g'), ''),
|
||
substring(source_file from '^ZY[0-9]{{2}}([0-9]{{10}})')
|
||
),
|
||
10,
|
||
'0'
|
||
),
|
||
10
|
||
)
|
||
WHERE (inpatient_no IS NULL OR BTRIM(inpatient_no) = '')
|
||
AND COALESCE(
|
||
LPAD(admission_count::text, 2, '0'),
|
||
substring(source_file from '^ZY([0-9]{{2}})[0-9]{{10}}')
|
||
) IS NOT NULL
|
||
AND COALESCE(
|
||
NULLIF(regexp_replace(COALESCE(front_page_medical_record_no, ''), '\\D', '', 'g'), ''),
|
||
NULLIF(regexp_replace(COALESCE(medical_record_no, ''), '\\D', '', 'g'), ''),
|
||
substring(source_file from '^ZY[0-9]{{2}}([0-9]{{10}})')
|
||
) IS NOT NULL;
|
||
ALTER TABLE {table_name} DROP CONSTRAINT IF EXISTS {source_file_constraint};
|
||
{drop_inpatient_no_checks}
|
||
DELETE FROM {table_name}
|
||
WHERE NULLIF(BTRIM(inpatient_no), '') IS NULL;
|
||
UPDATE {table_name}
|
||
SET inpatient_no = BTRIM(inpatient_no)
|
||
WHERE inpatient_no <> BTRIM(inpatient_no);
|
||
WITH ranked AS (
|
||
SELECT
|
||
id,
|
||
ROW_NUMBER() OVER (PARTITION BY BTRIM(inpatient_no) ORDER BY id DESC) AS duplicate_rank
|
||
FROM {table_name}
|
||
WHERE NULLIF(BTRIM(inpatient_no), '') IS NOT NULL
|
||
)
|
||
DELETE FROM {table_name} p
|
||
USING ranked
|
||
WHERE p.id = ranked.id
|
||
AND ranked.duplicate_rank > 1;
|
||
ALTER TABLE {table_name} ALTER COLUMN inpatient_no SET NOT NULL;
|
||
ALTER TABLE {table_name} DROP CONSTRAINT IF EXISTS {inpatient_no_constraint};
|
||
ALTER TABLE {table_name} ADD CONSTRAINT {inpatient_no_constraint} UNIQUE (inpatient_no);
|
||
DO $$
|
||
BEGIN
|
||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = {pg_literal(f"ck_{args.pg_table}_inpatient_no_required")}) THEN
|
||
ALTER TABLE {table_name} ADD CONSTRAINT {inpatient_no_required_constraint} CHECK (NULLIF(BTRIM(inpatient_no), '') IS NOT NULL);
|
||
END IF;
|
||
END $$;
|
||
COMMENT ON TABLE {table_name} IS '患者住院病案首页结构化宽表，由PDF首页解析程序生成并支持人工复核。';
|
||
{table_comments}
|
||
|
||
CREATE TABLE IF NOT EXISTS {list_table_name} (
|
||
record_id BIGSERIAL PRIMARY KEY,
|
||
batch_name TEXT NOT NULL DEFAULT 'Patient_FrontPages',
|
||
major_department TEXT NOT NULL DEFAULT '',
|
||
sub_department TEXT NOT NULL DEFAULT '',
|
||
source_folder TEXT NOT NULL DEFAULT 'Patient_FrontPages',
|
||
image_path TEXT NOT NULL DEFAULT '',
|
||
image_name TEXT NOT NULL DEFAULT '',
|
||
image_row_no INTEGER NOT NULL DEFAULT 0,
|
||
patient_name TEXT NOT NULL DEFAULT '',
|
||
gender TEXT,
|
||
age TEXT,
|
||
inpatient_no TEXT NOT NULL,
|
||
diagnosis TEXT,
|
||
admission_time TEXT,
|
||
last_write_time TEXT,
|
||
hospital_days INTEGER,
|
||
discharge_time TEXT,
|
||
postoperative_days TEXT,
|
||
review_status TEXT NOT NULL DEFAULT '首页自动关联',
|
||
review_notes TEXT,
|
||
manual_corrected BOOLEAN NOT NULL DEFAULT false,
|
||
imported_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
ALTER TABLE {list_table_name} ADD COLUMN IF NOT EXISTS has_front_page BOOLEAN NOT NULL DEFAULT false;
|
||
ALTER TABLE {list_table_name} ADD COLUMN IF NOT EXISTS front_page_id BIGINT;
|
||
ALTER TABLE {list_table_name} ADD COLUMN IF NOT EXISTS front_page_source_file TEXT;
|
||
{drop_list_inpatient_no_checks}
|
||
DELETE FROM {list_table_name}
|
||
WHERE NULLIF(BTRIM(inpatient_no), '') IS NULL;
|
||
UPDATE {list_table_name}
|
||
SET inpatient_no = BTRIM(inpatient_no)
|
||
WHERE inpatient_no <> BTRIM(inpatient_no);
|
||
WITH ranked AS (
|
||
SELECT
|
||
record_id,
|
||
ROW_NUMBER() OVER (PARTITION BY BTRIM(inpatient_no) ORDER BY record_id DESC) AS duplicate_rank
|
||
FROM {list_table_name}
|
||
WHERE NULLIF(BTRIM(inpatient_no), '') IS NOT NULL
|
||
)
|
||
DELETE FROM {list_table_name} pl
|
||
USING ranked
|
||
WHERE pl.record_id = ranked.record_id
|
||
AND ranked.duplicate_rank > 1;
|
||
ALTER TABLE {list_table_name} ALTER COLUMN inpatient_no SET NOT NULL;
|
||
DO $$
|
||
BEGIN
|
||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'ck_patient_lists_inpatient_no_required') THEN
|
||
ALTER TABLE {list_table_name} ADD CONSTRAINT {list_inpatient_no_required_constraint} CHECK (NULLIF(BTRIM(inpatient_no), '') IS NOT NULL);
|
||
END IF;
|
||
END $$;
|
||
COMMENT ON COLUMN {list_table_name}.has_front_page IS '是否有患者首页：由Patient_FrontPages按住院号自动联动。';
|
||
COMMENT ON COLUMN {list_table_name}.front_page_id IS '关联的Patient_FrontPages.id。';
|
||
COMMENT ON COLUMN {list_table_name}.front_page_source_file IS '关联患者首页PDF文件名。';
|
||
CREATE UNIQUE INDEX IF NOT EXISTS uq_patient_lists_inpatient_no ON {list_table_name}(inpatient_no);
|
||
|
||
CREATE OR REPLACE FUNCTION {dedupe_trigger_function_name}()
|
||
RETURNS trigger
|
||
LANGUAGE plpgsql
|
||
AS $dedupe_trigger$
|
||
DECLARE
|
||
existing_id BIGINT;
|
||
BEGIN
|
||
NEW.inpatient_no := BTRIM(NEW.inpatient_no);
|
||
IF NULLIF(NEW.inpatient_no, '') IS NULL THEN
|
||
RETURN NEW;
|
||
END IF;
|
||
|
||
IF TG_OP = 'INSERT' THEN
|
||
SELECT id
|
||
INTO existing_id
|
||
FROM {table_name}
|
||
WHERE BTRIM(inpatient_no) = NEW.inpatient_no
|
||
ORDER BY id DESC
|
||
LIMIT 1;
|
||
|
||
IF existing_id IS NOT NULL THEN
|
||
DELETE FROM {table_name}
|
||
WHERE BTRIM(inpatient_no) = NEW.inpatient_no
|
||
AND id <> existing_id;
|
||
|
||
UPDATE {table_name}
|
||
SET {dedupe_update_values}
|
||
WHERE id = existing_id;
|
||
|
||
RETURN NULL;
|
||
END IF;
|
||
END IF;
|
||
|
||
DELETE FROM {table_name}
|
||
WHERE BTRIM(inpatient_no) = NEW.inpatient_no
|
||
AND id <> NEW.id;
|
||
|
||
RETURN NEW;
|
||
END;
|
||
$dedupe_trigger$;
|
||
|
||
DROP TRIGGER IF EXISTS {dedupe_trigger_name} ON {table_name};
|
||
|
||
CREATE OR REPLACE FUNCTION {trigger_function_name}()
|
||
RETURNS trigger
|
||
LANGUAGE plpgsql
|
||
AS $trigger$
|
||
BEGIN
|
||
IF TG_OP = 'DELETE' THEN
|
||
IF NULLIF(BTRIM(OLD.inpatient_no), '') IS NOT NULL THEN
|
||
UPDATE {list_table_name} AS pl
|
||
SET has_front_page = false,
|
||
front_page_id = NULL,
|
||
front_page_source_file = NULL,
|
||
imported_at = now()
|
||
WHERE pl.inpatient_no = BTRIM(OLD.inpatient_no)
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM {table_name} fp
|
||
WHERE BTRIM(fp.inpatient_no) = BTRIM(OLD.inpatient_no)
|
||
);
|
||
END IF;
|
||
RETURN OLD;
|
||
END IF;
|
||
|
||
IF TG_OP = 'UPDATE'
|
||
AND NULLIF(BTRIM(OLD.inpatient_no), '') IS NOT NULL
|
||
AND BTRIM(OLD.inpatient_no) IS DISTINCT FROM BTRIM(NEW.inpatient_no) THEN
|
||
UPDATE {list_table_name} AS pl
|
||
SET has_front_page = false,
|
||
front_page_id = NULL,
|
||
front_page_source_file = NULL,
|
||
imported_at = now()
|
||
WHERE pl.inpatient_no = BTRIM(OLD.inpatient_no)
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM {table_name} fp
|
||
WHERE BTRIM(fp.inpatient_no) = BTRIM(OLD.inpatient_no)
|
||
);
|
||
END IF;
|
||
|
||
IF NULLIF(BTRIM(NEW.inpatient_no), '') IS NOT NULL THEN
|
||
INSERT INTO {list_table_name} (
|
||
batch_name, major_department, sub_department, source_folder, image_path, image_name,
|
||
image_row_no, patient_name, gender, age, inpatient_no, diagnosis, admission_time,
|
||
hospital_days, discharge_time, review_status, review_notes, manual_corrected,
|
||
has_front_page, front_page_id, front_page_source_file, imported_at
|
||
)
|
||
VALUES (
|
||
'Patient_FrontPages',
|
||
COALESCE(NEW.major_department, ''),
|
||
COALESCE(NEW.discharge_dept, NEW.admission_dept, ''),
|
||
'Patient_FrontPages',
|
||
COALESCE(NEW.source_file, ''),
|
||
COALESCE(NEW.source_file, ''),
|
||
0,
|
||
COALESCE(NEW.patient_name, ''),
|
||
NEW.gender,
|
||
NEW.age,
|
||
BTRIM(NEW.inpatient_no),
|
||
NEW.primary_diagnosis,
|
||
to_char(NEW.admission_time, 'YYYY-MM-DD HH24:MI:SS'),
|
||
NEW.hospital_days,
|
||
to_char(NEW.discharge_time, 'YYYY-MM-DD HH24:MI:SS'),
|
||
'首页自动关联',
|
||
'由Patient_FrontPages触发器按住院号自动关联',
|
||
COALESCE(NEW.manual_corrected, false),
|
||
true,
|
||
NEW.id,
|
||
NEW.source_file,
|
||
now()
|
||
)
|
||
ON CONFLICT (inpatient_no) DO UPDATE SET
|
||
has_front_page = true,
|
||
front_page_id = EXCLUDED.front_page_id,
|
||
front_page_source_file = EXCLUDED.front_page_source_file,
|
||
patient_name = COALESCE(NULLIF(EXCLUDED.patient_name, ''), {list_table_name}.patient_name),
|
||
gender = EXCLUDED.gender,
|
||
age = EXCLUDED.age,
|
||
major_department = EXCLUDED.major_department,
|
||
sub_department = EXCLUDED.sub_department,
|
||
manual_corrected = EXCLUDED.manual_corrected,
|
||
imported_at = now();
|
||
END IF;
|
||
|
||
RETURN NEW;
|
||
END;
|
||
$trigger$;
|
||
|
||
DROP TRIGGER IF EXISTS {trigger_name} ON {table_name};
|
||
|
||
CREATE TEMP TABLE patient_front_pages_import (
|
||
{import_column_defs}
|
||
);
|
||
|
||
\\copy patient_front_pages_import({import_columns}) FROM '{escaped_csv}' WITH (FORMAT csv, HEADER true)
|
||
|
||
DELETE FROM patient_front_pages_import
|
||
WHERE NULLIF(BTRIM(inpatient_no), '') IS NULL;
|
||
|
||
INSERT INTO {table_name} (
|
||
{import_columns}
|
||
)
|
||
SELECT
|
||
{select_values}
|
||
FROM (
|
||
SELECT DISTINCT ON (BTRIM(inpatient_no)) *
|
||
FROM patient_front_pages_import
|
||
WHERE NULLIF(BTRIM(inpatient_no), '') IS NOT NULL
|
||
ORDER BY BTRIM(inpatient_no), ctid DESC
|
||
) patient_front_pages_import
|
||
ON CONFLICT (inpatient_no) DO UPDATE SET
|
||
{update_values};
|
||
|
||
WITH front_pages AS (
|
||
SELECT DISTINCT ON (BTRIM(inpatient_no))
|
||
id,
|
||
BTRIM(inpatient_no) AS inpatient_no,
|
||
source_file,
|
||
COALESCE(patient_name, '') AS patient_name,
|
||
gender,
|
||
age,
|
||
COALESCE(major_department, '') AS major_department,
|
||
COALESCE(discharge_dept, admission_dept, '') AS sub_department,
|
||
primary_diagnosis,
|
||
admission_time,
|
||
discharge_time,
|
||
hospital_days,
|
||
manual_corrected
|
||
FROM {table_name}
|
||
WHERE NULLIF(BTRIM(inpatient_no), '') IS NOT NULL
|
||
ORDER BY BTRIM(inpatient_no), id DESC
|
||
)
|
||
INSERT INTO {list_table_name} (
|
||
batch_name, major_department, sub_department, source_folder, image_path, image_name,
|
||
image_row_no, patient_name, gender, age, inpatient_no, diagnosis, admission_time,
|
||
hospital_days, discharge_time, review_status, review_notes, manual_corrected,
|
||
has_front_page, front_page_id, front_page_source_file, imported_at
|
||
)
|
||
SELECT
|
||
'Patient_FrontPages',
|
||
major_department,
|
||
sub_department,
|
||
'Patient_FrontPages',
|
||
source_file,
|
||
source_file,
|
||
0,
|
||
patient_name,
|
||
gender,
|
||
age,
|
||
inpatient_no,
|
||
primary_diagnosis,
|
||
to_char(admission_time, 'YYYY-MM-DD HH24:MI:SS'),
|
||
hospital_days,
|
||
to_char(discharge_time, 'YYYY-MM-DD HH24:MI:SS'),
|
||
'首页自动关联',
|
||
'由Patient_FrontPages按住院号自动关联',
|
||
manual_corrected,
|
||
true,
|
||
id,
|
||
source_file,
|
||
now()
|
||
FROM front_pages
|
||
ON CONFLICT (inpatient_no) DO UPDATE SET
|
||
has_front_page = true,
|
||
front_page_id = EXCLUDED.front_page_id,
|
||
front_page_source_file = EXCLUDED.front_page_source_file,
|
||
patient_name = COALESCE(NULLIF(EXCLUDED.patient_name, ''), {list_table_name}.patient_name),
|
||
gender = EXCLUDED.gender,
|
||
age = EXCLUDED.age,
|
||
major_department = EXCLUDED.major_department,
|
||
sub_department = EXCLUDED.sub_department,
|
||
manual_corrected = EXCLUDED.manual_corrected,
|
||
imported_at = now();
|
||
|
||
UPDATE {list_table_name} AS pl
|
||
SET has_front_page = false,
|
||
front_page_id = NULL,
|
||
front_page_source_file = NULL,
|
||
imported_at = now()
|
||
WHERE has_front_page IS TRUE
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM {table_name} fp
|
||
WHERE BTRIM(fp.inpatient_no) = pl.inpatient_no
|
||
);
|
||
|
||
CREATE TRIGGER {dedupe_trigger_name}
|
||
BEFORE INSERT OR UPDATE OF inpatient_no ON {table_name}
|
||
FOR EACH ROW EXECUTE FUNCTION {dedupe_trigger_function_name}();
|
||
|
||
CREATE TRIGGER {trigger_name}
|
||
AFTER INSERT OR UPDATE OR DELETE ON {table_name}
|
||
FOR EACH ROW EXECUTE FUNCTION {trigger_function_name}();
|
||
""".strip()
|
||
+ "\n",
|
||
encoding="utf-8",
|
||
)
|
||
|
||
completed = subprocess.run(
|
||
["psql", "--no-password", "--file", str(sql_path)],
|
||
check=False,
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
text=True,
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
env=connection_env,
|
||
)
|
||
if completed.returncode != 0:
|
||
raise RuntimeError(completed.stderr.strip() or completed.stdout.strip() or "PostgreSQL 写入失败")
|
||
print(f"PostgreSQL 写入完成：{len(records)} 条 -> {connection_env['PGHOST']}:{connection_env['PGPORT']}/{connection_env['PGDATABASE']}.{args.pg_table}")
|
||
|
||
|
||
def process(input_dir: Path, output_dir: Path, save_text: bool, args: argparse.Namespace) -> int:
|
||
pdf_files = sorted(input_dir.glob("*.pdf"))
|
||
if not pdf_files:
|
||
print(f"未找到 PDF：{input_dir}", file=sys.stderr)
|
||
return 2
|
||
|
||
department_rules = load_department_rules(args.department_rules.resolve()) if args.department_rules else {}
|
||
if department_rules:
|
||
print(f"已加载科室分类规则：{args.department_rules}")
|
||
|
||
records: list[dict[str, Any]] = []
|
||
failures: list[dict[str, str]] = []
|
||
for index, pdf_path in enumerate(pdf_files, start=1):
|
||
print(f"[{index}/{len(pdf_files)}] 处理 {pdf_path.name}")
|
||
try:
|
||
record, _text = parse_record(pdf_path, department_rules=department_rules, args=args)
|
||
records.append(record)
|
||
except Exception as exc: # noqa: BLE001 - 批处理需要继续处理下一份
|
||
failures.append({"源文件": pdf_path.name, "错误": str(exc)})
|
||
|
||
write_outputs(records, output_dir, save_text=save_text)
|
||
if args.write_postgres:
|
||
write_postgres(records, args)
|
||
|
||
if failures:
|
||
fail_path = output_dir / "04_复核与人工校正" / "患者首页_处理失败.csv"
|
||
with fail_path.open("w", encoding="utf-8-sig", newline="") as fp:
|
||
writer = csv.DictWriter(fp, fieldnames=["源文件", "错误"])
|
||
writer.writeheader()
|
||
writer.writerows(failures)
|
||
|
||
print(f"完成：成功 {len(records)} 份，失败 {len(failures)} 份。结果目录：{output_dir}")
|
||
return 1 if failures else 0
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
parser = argparse.ArgumentParser(description="批量解析患者住院病案首页 PDF，输出 JSONL/CSV/单份 JSON。")
|
||
parser.add_argument("-i", "--input-dir", type=Path, default=DEFAULT_INPUT_DIR, help="PDF 输入目录")
|
||
parser.add_argument("-o", "--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR, help="结果输出目录")
|
||
parser.add_argument("--no-text", action="store_true", help="不额外保存 pdftotext 抽取出的 txt")
|
||
parser.add_argument("--write-postgres", action="store_true", help="同时写入 PostgreSQL 表")
|
||
parser.add_argument("--pg-host", default=os.environ.get("PGHOST", ""), help="PostgreSQL 主机；也可用 PGHOST")
|
||
parser.add_argument("--pg-port", default=os.environ.get("PGPORT", "5432"), help="PostgreSQL 端口；也可用 PGPORT")
|
||
parser.add_argument("--pg-database", default=os.environ.get("PGDATABASE", ""), help="PostgreSQL 数据库；也可用 PGDATABASE")
|
||
parser.add_argument("--pg-user", default=os.environ.get("PGUSER", ""), help="PostgreSQL 用户名；也可用 PGUSER")
|
||
parser.add_argument("--pg-password", default=os.environ.get("PGPASSWORD", ""), help="PostgreSQL 密码；也可用 PGPASSWORD")
|
||
parser.add_argument("--pg-table", default=os.environ.get("PG_PATIENT_TABLE", DEFAULT_PG_TABLE), help="PostgreSQL 表名")
|
||
parser.add_argument("--department-rules", type=Path, default=DEFAULT_DEPARTMENT_RULE_PATH, help="科室到大科室分类规则 JSON")
|
||
parser.add_argument(
|
||
"--text-source",
|
||
choices=["auto", "pdftotext", "mineru"],
|
||
default=os.environ.get("PATIENT_FRONT_TEXT_SOURCE", "auto"),
|
||
help="PDF文本抽取方式：auto先pdftotext异常时再Mineru，pdftotext只用本地转换，mineru强制PDF转Markdown",
|
||
)
|
||
parser.add_argument("--mineru-client", type=Path, default=DEFAULT_MINERU_CLIENT_PATH, help="Mineru PDF转Markdown客户端脚本路径")
|
||
parser.add_argument("--mineru-md-dir", type=Path, default=DEFAULT_MINERU_MD_DIR, help="Mineru Markdown输出目录")
|
||
parser.add_argument("--mineru-url", default=DEFAULT_MINERU_URL, help="Mineru API服务地址；也可用 MINERU_URL")
|
||
parser.add_argument("--mineru-sync", action="store_true", help="调用Mineru时开启同步清理")
|
||
return parser
|
||
|
||
|
||
def main() -> int:
|
||
args = build_parser().parse_args()
|
||
return process(args.input_dir.resolve(), args.output_dir.resolve(), save_text=not args.no_text, args=args)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|