Import sanitized HIS processing tools
This commit is contained in:
247
患者列表处理/数据处理工作区/04_合并批次结果.py
Normal file
247
患者列表处理/数据处理工作区/04_合并批次结果.py
Normal file
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Consolidate patient archive result folders."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path("数据处理结果区")
|
||||
RESULT_SUFFIX = "-列表归档结果"
|
||||
INFO_DIR_NAME = "信息记录"
|
||||
|
||||
|
||||
def write_json(path: Path, data: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as file:
|
||||
for record in records:
|
||||
file.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def flatten_record(batch_name: str, record: dict[str, Any]) -> dict[str, Any]:
|
||||
patient = record["患者信息"]
|
||||
image = record["图片信息"]
|
||||
review = record["复核"]
|
||||
return {
|
||||
"处理批次": batch_name,
|
||||
"大科室": record["大科室"],
|
||||
"子科室": record["子科室"],
|
||||
"来源文件夹": record["来源文件夹"],
|
||||
"图片路径": image["图片路径"],
|
||||
"图片名": image["图片名"],
|
||||
"图片内行号": image["图片内行号"],
|
||||
"拼接组序号": image["拼接组序号"],
|
||||
"OCR请求ID": image.get("OCR请求ID", ""),
|
||||
"姓名": patient["姓名"],
|
||||
"性别": patient["性别"],
|
||||
"年龄": patient["年龄"],
|
||||
"住院号": patient["住院号"],
|
||||
"诊断": patient["诊断"],
|
||||
"入院时间": patient["入院时间"],
|
||||
"最后书写时间": patient["最后书写时间"],
|
||||
"住院天数": patient["住院天数"],
|
||||
"出院时间": patient["出院时间"],
|
||||
"手术后天数": patient["手术后天数"],
|
||||
"复核状态": review["状态"],
|
||||
"复核提示": review_note_text(review),
|
||||
"人工修正": bool(review.get("人工修正")),
|
||||
}
|
||||
|
||||
|
||||
def review_note_text(review: dict[str, Any]) -> str:
|
||||
notes = [str(item) for item in review.get("提示", []) if str(item) and str(item) != "缺少出院时间"]
|
||||
if review.get("人工备注"):
|
||||
notes.append(f"人工备注: {review['人工备注']}")
|
||||
return "；".join(notes)
|
||||
|
||||
|
||||
def record_quality_rank(record: dict[str, Any]) -> tuple[int, int, int, int]:
|
||||
patient = record["患者信息"]
|
||||
review = record["复核"]
|
||||
review_ok = 0 if review.get("状态") == "需人工复核" else 1
|
||||
manual_corrected = 1 if review.get("人工修正") else 0
|
||||
date_count = int(bool(patient.get("入院时间"))) + int(bool(patient.get("出院时间")))
|
||||
filled_count = sum(1 for value in patient.values() if value not in ("", None))
|
||||
return (review_ok, manual_corrected, date_count, filled_count)
|
||||
|
||||
|
||||
def summarize_record_for_duplicate(record: dict[str, Any]) -> dict[str, Any]:
|
||||
patient = record["患者信息"]
|
||||
image = record["图片信息"]
|
||||
review = record["复核"]
|
||||
return {
|
||||
"处理批次": record["处理批次"],
|
||||
"大科室": record["大科室"],
|
||||
"子科室": record["子科室"],
|
||||
"来源文件夹": record["来源文件夹"],
|
||||
"图片路径": image["图片路径"],
|
||||
"图片名": image["图片名"],
|
||||
"图片内行号": image["图片内行号"],
|
||||
"姓名": patient.get("姓名", ""),
|
||||
"住院号": patient.get("住院号", ""),
|
||||
"入院时间": patient.get("入院时间", ""),
|
||||
"出院时间": patient.get("出院时间", ""),
|
||||
"复核状态": review.get("状态", ""),
|
||||
}
|
||||
|
||||
|
||||
def deduplicate_records_by_inpatient_no(
|
||||
records: list[dict[str, Any]],
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
kept_by_no: dict[str, dict[str, Any]] = {}
|
||||
kept_order: list[str] = []
|
||||
duplicates: list[dict[str, Any]] = []
|
||||
missing_inpatient_no_records: list[dict[str, Any]] = []
|
||||
for record in records:
|
||||
inpatient_no = str(record["患者信息"].get("住院号", "")).strip()
|
||||
if not inpatient_no:
|
||||
missing_inpatient_no_records.append(
|
||||
{
|
||||
"剔除记录": summarize_record_for_duplicate(record),
|
||||
"规则": "住院号为空，未纳入合并总表和数据库",
|
||||
}
|
||||
)
|
||||
continue
|
||||
if inpatient_no not in kept_by_no:
|
||||
kept_order.append(inpatient_no)
|
||||
else:
|
||||
previous = kept_by_no[inpatient_no]
|
||||
duplicates.append(
|
||||
{
|
||||
"住院号": inpatient_no,
|
||||
"保留记录": summarize_record_for_duplicate(record),
|
||||
"剔除记录": summarize_record_for_duplicate(previous),
|
||||
"规则": "住院号重复，后出现记录覆盖先出现记录",
|
||||
}
|
||||
)
|
||||
kept_by_no[inpatient_no] = record
|
||||
return [kept_by_no[inpatient_no] for inpatient_no in kept_order], duplicates, missing_inpatient_no_records
|
||||
|
||||
|
||||
def slim_department_summary(batch_name: str, item: dict[str, Any]) -> dict[str, Any]:
|
||||
raw_tips = item.get("提示", [])
|
||||
if isinstance(raw_tips, list):
|
||||
tips = "；".join(str(tip) for tip in raw_tips)
|
||||
else:
|
||||
tips = str(raw_tips)
|
||||
return {
|
||||
"处理批次": batch_name,
|
||||
"来源文件夹": item["来源文件夹"],
|
||||
"大科室": item["大科室"],
|
||||
"子科室": item["子科室"],
|
||||
"图片数": item["图片数"],
|
||||
"记录数": item["记录数"],
|
||||
"拼接组数": item["拼接组数"],
|
||||
"提示": tips,
|
||||
}
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
if not rows:
|
||||
return
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as file:
|
||||
writer = csv.DictWriter(file, fieldnames=list(rows[0].keys()))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ROOT.mkdir(exist_ok=True)
|
||||
raw_records: list[dict[str, Any]] = []
|
||||
batch_summaries: list[dict[str, Any]] = []
|
||||
|
||||
for result_dir in sorted(ROOT.rglob(f"*{RESULT_SUFFIX}")):
|
||||
archive_path = result_dir / "患者列表_结构化.json"
|
||||
review_path = result_dir / "复核报告.json"
|
||||
if not archive_path.exists():
|
||||
continue
|
||||
batch_name = result_dir.name[: -len(RESULT_SUFFIX)]
|
||||
archive = json.loads(archive_path.read_text(encoding="utf-8"))
|
||||
review = json.loads(review_path.read_text(encoding="utf-8")) if review_path.exists() else {}
|
||||
department_summaries = [
|
||||
slim_department_summary(batch_name, item) for item in archive.get("科室汇总", [])
|
||||
]
|
||||
summary = {
|
||||
"处理批次": batch_name,
|
||||
"结果目录": str(result_dir),
|
||||
**archive.get("汇总", {}),
|
||||
"科室归类": archive.get("科室归类", []),
|
||||
"科室汇总": department_summaries,
|
||||
"需人工复核记录数": len(review.get("需人工复核记录", [])),
|
||||
"人工修正记录数": len(review.get("人工修正记录", [])),
|
||||
}
|
||||
batch_summaries.append(summary)
|
||||
|
||||
summary_dir = result_dir / INFO_DIR_NAME
|
||||
write_json(summary_dir / "汇总.json", {**summary, "科室汇总": department_summaries})
|
||||
write_csv(summary_dir / "科室汇总.csv", department_summaries)
|
||||
|
||||
for record in archive.get("患者记录", []):
|
||||
enriched = {"处理批次": batch_name, **record}
|
||||
raw_records.append(enriched)
|
||||
|
||||
all_records, duplicate_records, missing_inpatient_no_records = deduplicate_records_by_inpatient_no(raw_records)
|
||||
flat_records = [flatten_record(record["处理批次"], record) for record in all_records]
|
||||
kept_by_batch = Counter(record["处理批次"] for record in all_records)
|
||||
dropped_by_batch = Counter(item["剔除记录"]["处理批次"] for item in duplicate_records)
|
||||
missing_by_batch = Counter(item["剔除记录"]["处理批次"] for item in missing_inpatient_no_records)
|
||||
batch_level_duplicate_count = sum(int(item.get("重复住院号剔除记录数", 0)) for item in batch_summaries)
|
||||
batch_level_raw_count = sum(
|
||||
int(item.get("去重前患者记录数", int(item.get("患者记录数", 0)) + int(item.get("重复住院号剔除记录数", 0))))
|
||||
for item in batch_summaries
|
||||
)
|
||||
|
||||
for summary in batch_summaries:
|
||||
batch_name = summary["处理批次"]
|
||||
original_count = int(summary.get("患者记录数", 0))
|
||||
summary["原始患者记录数"] = original_count
|
||||
summary["合并阶段重复住院号剔除记录数"] = int(dropped_by_batch.get(batch_name, 0))
|
||||
summary["合并阶段缺少住院号剔除记录数"] = int(missing_by_batch.get(batch_name, 0))
|
||||
summary["患者记录数"] = int(kept_by_batch.get(batch_name, 0))
|
||||
write_json(Path(summary["结果目录"]) / INFO_DIR_NAME / "汇总.json", summary)
|
||||
|
||||
global_summary = {
|
||||
"批次数": len(batch_summaries),
|
||||
"总图片数": sum(int(item.get("图片数", 0)) for item in batch_summaries),
|
||||
"批次内去重前患者记录数": batch_level_raw_count,
|
||||
"批次内重复住院号剔除记录数": batch_level_duplicate_count,
|
||||
"合并前患者记录数": len(raw_records),
|
||||
"总患者记录数": len(all_records),
|
||||
"合并阶段重复住院号剔除记录数": len(duplicate_records),
|
||||
"合并阶段缺少住院号剔除记录数": len(missing_inpatient_no_records),
|
||||
"重复住院号剔除记录数": batch_level_duplicate_count + len(duplicate_records),
|
||||
"需人工复核记录数": sum(int(item.get("需人工复核记录数", 0)) for item in batch_summaries),
|
||||
"人工修正记录数": sum(int(item.get("人工修正记录数", 0)) for item in batch_summaries),
|
||||
"批次汇总": batch_summaries,
|
||||
}
|
||||
write_json(ROOT / INFO_DIR_NAME / "全局汇总.json", global_summary)
|
||||
write_csv(ROOT / INFO_DIR_NAME / "批次汇总.csv", batch_summaries)
|
||||
write_json(
|
||||
ROOT / "合并_患者列表_结构化.json",
|
||||
{
|
||||
"汇总": global_summary,
|
||||
"重复住院号剔除记录": duplicate_records,
|
||||
"缺少住院号剔除记录": missing_inpatient_no_records,
|
||||
"患者记录": all_records,
|
||||
},
|
||||
)
|
||||
write_jsonl(ROOT / "合并_患者列表_记录.jsonl", all_records)
|
||||
write_csv(ROOT / "合并_患者列表_记录.csv", flat_records)
|
||||
write_json(ROOT / INFO_DIR_NAME / "重复住院号报告.json", duplicate_records)
|
||||
write_json(ROOT / INFO_DIR_NAME / "缺少住院号报告.json", missing_inpatient_no_records)
|
||||
print(json.dumps(global_summary, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user