248 lines
11 KiB
Python
248 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""Consolidate patient archive result folders."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import csv
|
||
import json
|
||
from collections import Counter
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
|
||
ROOT = Path("数据处理结果区")
|
||
RESULT_SUFFIX = "-列表归档结果"
|
||
INFO_DIR_NAME = "信息记录"
|
||
|
||
|
||
def write_json(path: Path, data: Any) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
|
||
|
||
def write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
with path.open("w", encoding="utf-8") as file:
|
||
for record in records:
|
||
file.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||
|
||
|
||
def flatten_record(batch_name: str, record: dict[str, Any]) -> dict[str, Any]:
|
||
patient = record["患者信息"]
|
||
image = record["图片信息"]
|
||
review = record["复核"]
|
||
return {
|
||
"处理批次": batch_name,
|
||
"大科室": record["大科室"],
|
||
"子科室": record["子科室"],
|
||
"来源文件夹": record["来源文件夹"],
|
||
"图片路径": image["图片路径"],
|
||
"图片名": image["图片名"],
|
||
"图片内行号": image["图片内行号"],
|
||
"拼接组序号": image["拼接组序号"],
|
||
"OCR请求ID": image.get("OCR请求ID", ""),
|
||
"姓名": patient["姓名"],
|
||
"性别": patient["性别"],
|
||
"年龄": patient["年龄"],
|
||
"住院号": patient["住院号"],
|
||
"诊断": patient["诊断"],
|
||
"入院时间": patient["入院时间"],
|
||
"最后书写时间": patient["最后书写时间"],
|
||
"住院天数": patient["住院天数"],
|
||
"出院时间": patient["出院时间"],
|
||
"手术后天数": patient["手术后天数"],
|
||
"复核状态": review["状态"],
|
||
"复核提示": review_note_text(review),
|
||
"人工修正": bool(review.get("人工修正")),
|
||
}
|
||
|
||
|
||
def review_note_text(review: dict[str, Any]) -> str:
|
||
notes = [str(item) for item in review.get("提示", []) if str(item) and str(item) != "缺少出院时间"]
|
||
if review.get("人工备注"):
|
||
notes.append(f"人工备注: {review['人工备注']}")
|
||
return "；".join(notes)
|
||
|
||
|
||
def record_quality_rank(record: dict[str, Any]) -> tuple[int, int, int, int]:
|
||
patient = record["患者信息"]
|
||
review = record["复核"]
|
||
review_ok = 0 if review.get("状态") == "需人工复核" else 1
|
||
manual_corrected = 1 if review.get("人工修正") else 0
|
||
date_count = int(bool(patient.get("入院时间"))) + int(bool(patient.get("出院时间")))
|
||
filled_count = sum(1 for value in patient.values() if value not in ("", None))
|
||
return (review_ok, manual_corrected, date_count, filled_count)
|
||
|
||
|
||
def summarize_record_for_duplicate(record: dict[str, Any]) -> dict[str, Any]:
|
||
patient = record["患者信息"]
|
||
image = record["图片信息"]
|
||
review = record["复核"]
|
||
return {
|
||
"处理批次": record["处理批次"],
|
||
"大科室": record["大科室"],
|
||
"子科室": record["子科室"],
|
||
"来源文件夹": record["来源文件夹"],
|
||
"图片路径": image["图片路径"],
|
||
"图片名": image["图片名"],
|
||
"图片内行号": image["图片内行号"],
|
||
"姓名": patient.get("姓名", ""),
|
||
"住院号": patient.get("住院号", ""),
|
||
"入院时间": patient.get("入院时间", ""),
|
||
"出院时间": patient.get("出院时间", ""),
|
||
"复核状态": review.get("状态", ""),
|
||
}
|
||
|
||
|
||
def deduplicate_records_by_inpatient_no(
|
||
records: list[dict[str, Any]],
|
||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
|
||
kept_by_no: dict[str, dict[str, Any]] = {}
|
||
kept_order: list[str] = []
|
||
duplicates: list[dict[str, Any]] = []
|
||
missing_inpatient_no_records: list[dict[str, Any]] = []
|
||
for record in records:
|
||
inpatient_no = str(record["患者信息"].get("住院号", "")).strip()
|
||
if not inpatient_no:
|
||
missing_inpatient_no_records.append(
|
||
{
|
||
"剔除记录": summarize_record_for_duplicate(record),
|
||
"规则": "住院号为空，未纳入合并总表和数据库",
|
||
}
|
||
)
|
||
continue
|
||
if inpatient_no not in kept_by_no:
|
||
kept_order.append(inpatient_no)
|
||
else:
|
||
previous = kept_by_no[inpatient_no]
|
||
duplicates.append(
|
||
{
|
||
"住院号": inpatient_no,
|
||
"保留记录": summarize_record_for_duplicate(record),
|
||
"剔除记录": summarize_record_for_duplicate(previous),
|
||
"规则": "住院号重复，后出现记录覆盖先出现记录",
|
||
}
|
||
)
|
||
kept_by_no[inpatient_no] = record
|
||
return [kept_by_no[inpatient_no] for inpatient_no in kept_order], duplicates, missing_inpatient_no_records
|
||
|
||
|
||
def slim_department_summary(batch_name: str, item: dict[str, Any]) -> dict[str, Any]:
|
||
raw_tips = item.get("提示", [])
|
||
if isinstance(raw_tips, list):
|
||
tips = "；".join(str(tip) for tip in raw_tips)
|
||
else:
|
||
tips = str(raw_tips)
|
||
return {
|
||
"处理批次": batch_name,
|
||
"来源文件夹": item["来源文件夹"],
|
||
"大科室": item["大科室"],
|
||
"子科室": item["子科室"],
|
||
"图片数": item["图片数"],
|
||
"记录数": item["记录数"],
|
||
"拼接组数": item["拼接组数"],
|
||
"提示": tips,
|
||
}
|
||
|
||
|
||
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
||
if not rows:
|
||
return
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
with path.open("w", encoding="utf-8-sig", newline="") as file:
|
||
writer = csv.DictWriter(file, fieldnames=list(rows[0].keys()))
|
||
writer.writeheader()
|
||
writer.writerows(rows)
|
||
|
||
|
||
def main() -> None:
|
||
ROOT.mkdir(exist_ok=True)
|
||
raw_records: list[dict[str, Any]] = []
|
||
batch_summaries: list[dict[str, Any]] = []
|
||
|
||
for result_dir in sorted(ROOT.rglob(f"*{RESULT_SUFFIX}")):
|
||
archive_path = result_dir / "患者列表_结构化.json"
|
||
review_path = result_dir / "复核报告.json"
|
||
if not archive_path.exists():
|
||
continue
|
||
batch_name = result_dir.name[: -len(RESULT_SUFFIX)]
|
||
archive = json.loads(archive_path.read_text(encoding="utf-8"))
|
||
review = json.loads(review_path.read_text(encoding="utf-8")) if review_path.exists() else {}
|
||
department_summaries = [
|
||
slim_department_summary(batch_name, item) for item in archive.get("科室汇总", [])
|
||
]
|
||
summary = {
|
||
"处理批次": batch_name,
|
||
"结果目录": str(result_dir),
|
||
**archive.get("汇总", {}),
|
||
"科室归类": archive.get("科室归类", []),
|
||
"科室汇总": department_summaries,
|
||
"需人工复核记录数": len(review.get("需人工复核记录", [])),
|
||
"人工修正记录数": len(review.get("人工修正记录", [])),
|
||
}
|
||
batch_summaries.append(summary)
|
||
|
||
summary_dir = result_dir / INFO_DIR_NAME
|
||
write_json(summary_dir / "汇总.json", {**summary, "科室汇总": department_summaries})
|
||
write_csv(summary_dir / "科室汇总.csv", department_summaries)
|
||
|
||
for record in archive.get("患者记录", []):
|
||
enriched = {"处理批次": batch_name, **record}
|
||
raw_records.append(enriched)
|
||
|
||
all_records, duplicate_records, missing_inpatient_no_records = deduplicate_records_by_inpatient_no(raw_records)
|
||
flat_records = [flatten_record(record["处理批次"], record) for record in all_records]
|
||
kept_by_batch = Counter(record["处理批次"] for record in all_records)
|
||
dropped_by_batch = Counter(item["剔除记录"]["处理批次"] for item in duplicate_records)
|
||
missing_by_batch = Counter(item["剔除记录"]["处理批次"] for item in missing_inpatient_no_records)
|
||
batch_level_duplicate_count = sum(int(item.get("重复住院号剔除记录数", 0)) for item in batch_summaries)
|
||
batch_level_raw_count = sum(
|
||
int(item.get("去重前患者记录数", int(item.get("患者记录数", 0)) + int(item.get("重复住院号剔除记录数", 0))))
|
||
for item in batch_summaries
|
||
)
|
||
|
||
for summary in batch_summaries:
|
||
batch_name = summary["处理批次"]
|
||
original_count = int(summary.get("患者记录数", 0))
|
||
summary["原始患者记录数"] = original_count
|
||
summary["合并阶段重复住院号剔除记录数"] = int(dropped_by_batch.get(batch_name, 0))
|
||
summary["合并阶段缺少住院号剔除记录数"] = int(missing_by_batch.get(batch_name, 0))
|
||
summary["患者记录数"] = int(kept_by_batch.get(batch_name, 0))
|
||
write_json(Path(summary["结果目录"]) / INFO_DIR_NAME / "汇总.json", summary)
|
||
|
||
global_summary = {
|
||
"批次数": len(batch_summaries),
|
||
"总图片数": sum(int(item.get("图片数", 0)) for item in batch_summaries),
|
||
"批次内去重前患者记录数": batch_level_raw_count,
|
||
"批次内重复住院号剔除记录数": batch_level_duplicate_count,
|
||
"合并前患者记录数": len(raw_records),
|
||
"总患者记录数": len(all_records),
|
||
"合并阶段重复住院号剔除记录数": len(duplicate_records),
|
||
"合并阶段缺少住院号剔除记录数": len(missing_inpatient_no_records),
|
||
"重复住院号剔除记录数": batch_level_duplicate_count + len(duplicate_records),
|
||
"需人工复核记录数": sum(int(item.get("需人工复核记录数", 0)) for item in batch_summaries),
|
||
"人工修正记录数": sum(int(item.get("人工修正记录数", 0)) for item in batch_summaries),
|
||
"批次汇总": batch_summaries,
|
||
}
|
||
write_json(ROOT / INFO_DIR_NAME / "全局汇总.json", global_summary)
|
||
write_csv(ROOT / INFO_DIR_NAME / "批次汇总.csv", batch_summaries)
|
||
write_json(
|
||
ROOT / "合并_患者列表_结构化.json",
|
||
{
|
||
"汇总": global_summary,
|
||
"重复住院号剔除记录": duplicate_records,
|
||
"缺少住院号剔除记录": missing_inpatient_no_records,
|
||
"患者记录": all_records,
|
||
},
|
||
)
|
||
write_jsonl(ROOT / "合并_患者列表_记录.jsonl", all_records)
|
||
write_csv(ROOT / "合并_患者列表_记录.csv", flat_records)
|
||
write_json(ROOT / INFO_DIR_NAME / "重复住院号报告.json", duplicate_records)
|
||
write_json(ROOT / INFO_DIR_NAME / "缺少住院号报告.json", missing_inpatient_no_records)
|
||
print(json.dumps(global_summary, ensure_ascii=False, indent=2))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|