diff --git a/.gitignore b/.gitignore index d83120c..669b7c8 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,8 @@ UPP列表处理/数据处理工作区/03_人工复核修正.json UPP列表处理/数据处理工作区/06_PostgreSQL建表结构.sql *.dump *.sql.gz + +# STL/CT 实数据、逐病例明细和本地同步报告不提交 +UPP_STL处理/ +UPP_数据库构建/UPP_STL资产同步报告.json +UPP_数据库构建/UPP_STL文件family顺序明细.csv diff --git a/UPP_数据库构建/01_UPP资产索引建表.sql b/UPP_数据库构建/01_UPP资产索引建表.sql new file mode 100644 index 0000000..8076c06 --- /dev/null +++ b/UPP_数据库构建/01_UPP资产索引建表.sql @@ -0,0 +1,92 @@ +CREATE TABLE IF NOT EXISTS upp_exam_assets ( + ct_number text NOT NULL PRIMARY KEY, + list_present boolean NOT NULL DEFAULT false, + stl_present boolean NOT NULL DEFAULT false, + patient_name text, + patient_sex text, + patient_age text, + patient_id_masked text, + exam_date timestamptz, + task_created_at timestamptz, + exam_description text, + exam_device text, + algorithm_model text, + upp_status text, + list_record_count integer NOT NULL DEFAULT 0, + selected_list_record jsonb, + list_records jsonb NOT NULL DEFAULT '[]'::jsonb, + selected_source_case_dir text, + selected_source_stl_dir text, + processed_stl_dir text, + stl_case_name text, + stl_sequence_no integer, + stl_file_count integer NOT NULL DEFAULT 0, + stl_total_bytes bigint NOT NULL DEFAULT 0, + stl_files jsonb NOT NULL DEFAULT '[]'::jsonb, + stl_candidates jsonb NOT NULL DEFAULT '[]'::jsonb, + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT ck_upp_exam_assets_ct_number_present CHECK (btrim(ct_number) <> ''), + CONSTRAINT ck_upp_exam_assets_ct_number_format CHECK (ct_number ~ '^D?CT[0-9]{8,}$') +); + +DO $$ +BEGIN + IF to_regclass('public.upp_stl_files') IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'upp_stl_files' + AND column_name = 'file_names' + ) + THEN + CREATE TABLE IF NOT EXISTS upp_stl_files_row_detail_backup AS TABLE upp_stl_files; + DROP TABLE upp_stl_files; + END IF; +END $$; + +CREATE TABLE IF NOT EXISTS upp_stl_files ( + ct_number text NOT NULL PRIMARY KEY REFERENCES upp_exam_assets(ct_number) ON DELETE CASCADE, + file_count integer NOT NULL DEFAULT 0, + total_bytes bigint NOT NULL DEFAULT 0, + segment_names jsonb NOT NULL DEFAULT '[]'::jsonb, + segment_families jsonb NOT NULL DEFAULT '[]'::jsonb, + segment_categories jsonb NOT NULL DEFAULT '[]'::jsonb, + file_names jsonb NOT NULL DEFAULT '[]'::jsonb, + source_file_paths jsonb NOT NULL DEFAULT '[]'::jsonb, + processed_file_paths jsonb NOT NULL DEFAULT '[]'::jsonb, + files jsonb NOT NULL DEFAULT '[]'::jsonb, + updated_at timestamptz NOT NULL DEFAULT now() +); + +ALTER TABLE upp_exam_assets ALTER COLUMN ct_number SET NOT NULL; +ALTER TABLE upp_stl_files ALTER COLUMN ct_number SET NOT NULL; +ALTER TABLE upp_stl_files ADD COLUMN IF NOT EXISTS segment_families jsonb NOT NULL DEFAULT '[]'::jsonb; +ALTER TABLE upp_stl_files ADD COLUMN IF NOT EXISTS segment_categories jsonb NOT NULL DEFAULT '[]'::jsonb; + +CREATE INDEX IF NOT EXISTS idx_upp_exam_assets_list_present ON upp_exam_assets(list_present); +CREATE INDEX IF NOT EXISTS idx_upp_exam_assets_stl_present ON upp_exam_assets(stl_present); +CREATE INDEX IF NOT EXISTS idx_upp_exam_assets_patient_name ON upp_exam_assets(patient_name); +CREATE INDEX IF NOT EXISTS idx_upp_exam_assets_exam_date ON upp_exam_assets(exam_date); +CREATE INDEX IF NOT EXISTS idx_upp_exam_assets_list_records_gin ON upp_exam_assets USING gin (list_records); +CREATE INDEX IF NOT EXISTS idx_upp_exam_assets_stl_files_gin ON upp_exam_assets USING gin (stl_files); +CREATE INDEX IF NOT EXISTS idx_upp_stl_files_segment_names_gin ON upp_stl_files USING gin (segment_names); +CREATE INDEX IF NOT EXISTS idx_upp_stl_files_segment_families_gin ON upp_stl_files USING gin (segment_families); +CREATE INDEX IF NOT EXISTS idx_upp_stl_files_segment_categories_gin ON upp_stl_files USING gin (segment_categories); +CREATE INDEX IF NOT EXISTS idx_upp_stl_files_file_names_gin ON upp_stl_files USING gin (file_names); +CREATE INDEX IF NOT EXISTS idx_upp_stl_files_files_gin ON upp_stl_files USING gin (files); + +COMMENT ON TABLE upp_exam_assets IS 'UPP列表、STL重建结果、未来CT数据的CT号唯一资产索引'; +COMMENT ON COLUMN upp_exam_assets.ct_number IS 'CT检查号，唯一索引键'; +COMMENT ON COLUMN upp_exam_assets.list_records IS '来自UPP列表OCR的同CT号全部记录'; +COMMENT ON COLUMN upp_exam_assets.selected_list_record IS '同CT号列表记录中按任务创建时间/检查时间选择的新记录'; +COMMENT ON COLUMN upp_exam_assets.stl_candidates IS '同CT号全部候选STL目录，保留去重选择依据'; +COMMENT ON COLUMN upp_exam_assets.stl_files IS '最终选中STL目录的文件清单'; +COMMENT ON COLUMN upp_exam_assets.processed_stl_dir IS '规范化后的STL目录，目录名为CT号'; +COMMENT ON TABLE upp_stl_files IS 'UPP最终选中STL文件聚合表，每个CT号唯一一行'; +COMMENT ON COLUMN upp_stl_files.ct_number IS 'CT检查号，主键并关联upp_exam_assets.ct_number'; +COMMENT ON COLUMN upp_stl_files.segment_names IS '最终选中STL文件的分割名称JSON数组'; +COMMENT ON COLUMN upp_stl_files.segment_families IS '与segment_names同顺序的训练family JSON数组'; +COMMENT ON COLUMN upp_stl_files.segment_categories IS '与segment_names同顺序的粗分类JSON数组'; +COMMENT ON COLUMN upp_stl_files.file_names IS '最终选中STL文件名JSON数组'; +COMMENT ON COLUMN upp_stl_files.files IS '最终选中STL文件完整明细JSON数组'; diff --git a/UPP_数据库构建/02_同步UPP_STL资产.py b/UPP_数据库构建/02_同步UPP_STL资产.py new file mode 100755 index 0000000..5db62b2 --- /dev/null +++ b/UPP_数据库构建/02_同步UPP_STL资产.py @@ -0,0 +1,672 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""整理UPP STL目录，并把CT号唯一资产索引同步到PostgreSQL。""" + +from __future__ import annotations + +import argparse +import csv +import json +import os +import re +import shutil +import subprocess +import tempfile +import unicodedata +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any + + +BASE_DIR = Path(__file__).resolve().parents[1] +DEFAULT_STL_ROOT = BASE_DIR / "UPP_STL处理" / "待处理STL数据" +DEFAULT_PROCESSED_ROOT = BASE_DIR / "UPP_STL处理" / "已处理STL数据" +DEFAULT_LIST_JSON = BASE_DIR / "UPP列表处理" / "数据处理结果区" / "全量分片结果" / "合并_图片表格_结构化.json" +DEFAULT_SCHEMA = BASE_DIR / "UPP_数据库构建" / "01_UPP资产索引建表.sql" +DEFAULT_REPORT = BASE_DIR / "UPP_数据库构建" / "UPP_STL资产同步报告.json" +CT_PATTERN = re.compile(r"(D?CT\d{8,})", re.IGNORECASE) +VALID_CT_PATTERN = re.compile(r"^D?CT\d{8,}$") + + +ASSET_FIELDS = [ + "ct_number", + "list_present", + "stl_present", + "patient_name", + "patient_sex", + "patient_age", + "patient_id_masked", + "exam_date", + "task_created_at", + "exam_description", + "exam_device", + "algorithm_model", + "upp_status", + "list_record_count", + "selected_list_record", + "list_records", + "selected_source_case_dir", + "selected_source_stl_dir", + "processed_stl_dir", + "stl_case_name", + "stl_sequence_no", + "stl_file_count", + "stl_total_bytes", + "stl_files", + "stl_candidates", +] + +STL_FIELDS = [ + "ct_number", + "file_count", + "total_bytes", + "segment_names", + "segment_families", + "segment_categories", + "file_names", + "source_file_paths", + "processed_file_paths", + "files", +] + +MODEL_VALUES = {"肝胆模型", "泌尿模型", "胸外模型"} +DEVICE_VALUES = {"CT", "MR", "DR", "CR", "DX", "US", "XA", "NM", "PT"} + + +def normalize_text(value: Any) -> str: + if value is None: + return "" + text = unicodedata.normalize("NFKC", str(value)).replace("\u3000", " ") + return re.sub(r"\s+", " ", text).strip() + + +def normalize_ct(value: Any) -> str: + return re.sub(r"\s+", "", normalize_text(value)).upper() + + +def valid_ct(value: str) -> bool: + return bool(VALID_CT_PATTERN.fullmatch(value)) + + +def extract_ct(text: str) -> str: + match = CT_PATTERN.search(text) + if not match: + return "" + ct_number = normalize_ct(match.group(1)) + return ct_number if valid_ct(ct_number) else "" + + +def json_dump(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + + +def path_text(path: Path) -> str: + return str(path.expanduser().absolute()) + + +def parse_time(value: Any) -> datetime | None: + text = normalize_text(value) + if not text: + return None + for fmt in ("%Y-%m-%d %H:%M:%S", "%Y/%m/%d %H:%M:%S", "%Y-%m-%d", "%Y/%m/%d"): + try: + return datetime.strptime(text, fmt) + except ValueError: + pass + return None + + +def normalize_status(value: Any) -> str: + text = normalize_text(value).replace("\ufe0f", "") + compact = re.sub(r"\s+", "", text) + if not compact: + return "" + if "部分重建成功" in compact: + return "√ 部分重建成功" + if "重建失败" in compact or compact.startswith(("×", "✗")): + return "× 重建失败" + if "重建成功" in compact: + return "√ 重建成功" + if "已报告" in compact: + return "√ 已报告" + return text + + +def status_like(value: Any) -> bool: + text = normalize_text(value) + return bool(re.search(r"(重建成功|重建失败|部分重建成功|已报告)", text)) + + +def classify_segment(segment_name: str) -> tuple[str, str]: + if segment_name in {"liver", "liver_left", "liver_right"}: + return "肝脏主体", segment_name + if re.fullmatch(r"liver_segment_S[1-8]", segment_name): + return "肝段", segment_name + if segment_name in {"liver_artery", "liver_vein", "portal_vein", "bile_duct"}: + return "血管胆管", segment_name + if segment_name in {"pancreas", "spleen", "cholecyst"}: + return "腹部脏器", segment_name + if segment_name in {"skin", "rib", "vertebrae", "sternum", "hipbone", "sacrum"}: + return "体表骨骼", segment_name + if re.fullmatch(r"liver_tumor_\d+", segment_name): + return "肝脏肿瘤", "liver_tumor_*" + if re.fullmatch(r"liver_cyst_\d+", segment_name): + return "肝囊肿", "liver_cyst_*" + if re.fullmatch(r"liver_hemangioma_\d+", segment_name): + return "肝血管瘤", "liver_hemangioma_*" + if re.fullmatch(r"pancreas_tumor_\d+", segment_name): + return "胰腺肿瘤", "pancreas_tumor_*" + if re.fullmatch(r"Segment_\d+", segment_name): + return "未命名分割", "Segment_*" + return "其他", segment_name + + +def clean_record_info(info: dict[str, Any]) -> dict[str, Any]: + cleaned = dict(info) + exam_description = normalize_text(cleaned.get("检查描述")) + exam_device = normalize_text(cleaned.get("检查设备")) + algorithm_model = normalize_text(cleaned.get("算法模型")) + if exam_description in DEVICE_VALUES and exam_device in MODEL_VALUES and status_like(algorithm_model): + cleaned["检查描述"] = "" + cleaned["检查设备"] = exam_description + cleaned["算法模型"] = exam_device + cleaned["状态"] = normalize_status(algorithm_model) + elif exam_device in MODEL_VALUES and status_like(algorithm_model) and normalize_text(cleaned.get("状态")) in {"土", "士", "↓"}: + cleaned["检查设备"] = "CT" + cleaned["算法模型"] = exam_device + cleaned["状态"] = normalize_status(algorithm_model) + else: + cleaned["状态"] = normalize_status(cleaned.get("状态")) + return cleaned + + +def clean_record(record: dict[str, Any]) -> dict[str, Any]: + cleaned = dict(record) + info = cleaned.get("记录信息") + if isinstance(info, dict): + cleaned["记录信息"] = clean_record_info(info) + return cleaned + + +def sql_quote_path(path: Path) -> str: + return "'" + str(path.expanduser().absolute()).replace("'", "''") + "'" + + +@dataclass(frozen=True) +class StlCandidate: + ct_number: str + source_case_dir: Path + source_stl_dir: Path + sequence_no: int | None + files: tuple[Path, ...] + + @property + def file_count(self) -> int: + return len(self.files) + + @property + def total_bytes(self) -> int: + return sum(file.stat().st_size for file in self.files) + + def score(self) -> tuple[int, int, str, str]: + return ( + self.file_count, + self.sequence_no if self.sequence_no is not None else -1, + self.source_case_dir.name, + self.source_stl_dir.name, + ) + + def summary(self) -> dict[str, Any]: + return { + "source_case_dir": path_text(self.source_case_dir), + "source_stl_dir": path_text(self.source_stl_dir), + "case_name": self.source_case_dir.name, + "sequence_no": self.sequence_no, + "file_count": self.file_count, + "total_bytes": self.total_bytes, + } + + +def sequence_no(case_dir: Path, stl_dir: Path) -> int | None: + values: list[int] = [] + for pattern in (r"-(\d{3,6})_D?CT", r"^(\d{3,6})-STL$"): + for text in (case_dir.name, stl_dir.name): + values.extend(int(item) for item in re.findall(pattern, text, re.IGNORECASE)) + return max(values) if values else None + + +def scan_stl_candidates(stl_root: Path) -> tuple[dict[str, list[StlCandidate]], list[str]]: + candidates: dict[str, list[StlCandidate]] = {} + no_ct_dirs: list[str] = [] + for directory in sorted(stl_root.rglob("*")): + if not directory.is_dir(): + continue + files = tuple(sorted([item for item in directory.iterdir() if item.is_file() and item.suffix.lower() == ".stl"])) + if not files: + continue + ct_number = extract_ct("/".join(directory.relative_to(stl_root).parts)) + if not ct_number: + no_ct_dirs.append(path_text(directory)) + continue + candidate = StlCandidate( + ct_number=ct_number, + source_case_dir=directory.parent, + source_stl_dir=directory, + sequence_no=sequence_no(directory.parent, directory), + files=files, + ) + candidates.setdefault(ct_number, []).append(candidate) + return candidates, no_ct_dirs + + +def choose_candidates(candidates: dict[str, list[StlCandidate]]) -> dict[str, StlCandidate]: + return {ct_number: max(items, key=lambda item: item.score()) for ct_number, items in candidates.items()} + + +def clear_directory(directory: Path) -> None: + directory.mkdir(parents=True, exist_ok=True) + for child in directory.iterdir(): + if child.is_dir() and not child.is_symlink(): + shutil.rmtree(child) + else: + child.unlink() + + +def place_file(source: Path, destination: Path, mode: str) -> str: + destination.parent.mkdir(parents=True, exist_ok=True) + if mode == "symlink": + destination.symlink_to(source.expanduser().absolute()) + return "symlink" + if mode == "copy": + shutil.copy2(source, destination) + return "copy" + try: + os.link(source, destination) + return "hardlink" + except OSError: + shutil.copy2(source, destination) + return "copy" + + +def materialize_selected_stl( + selected: dict[str, StlCandidate], + processed_root: Path, + link_mode: str, + refresh_files: bool, +) -> tuple[dict[str, list[dict[str, Any]]], dict[str, str]]: + processed_files: dict[str, list[dict[str, Any]]] = {} + link_methods: dict[str, str] = {} + processed_root.mkdir(parents=True, exist_ok=True) + for ct_number, candidate in sorted(selected.items()): + ct_dir = processed_root / ct_number + if refresh_files: + clear_directory(ct_dir) + else: + ct_dir.mkdir(parents=True, exist_ok=True) + file_rows: list[dict[str, Any]] = [] + used_methods: set[str] = set() + for source in candidate.files: + destination = ct_dir / source.name + if destination.exists() or destination.is_symlink(): + destination.unlink() + method = place_file(source, destination, link_mode) + used_methods.add(method) + category, family = classify_segment(source.stem) + file_rows.append( + { + "segment_name": source.stem, + "family": family, + "category": category, + "file_name": source.name, + "source_file_path": path_text(source), + "processed_file_path": path_text(destination), + "size_bytes": source.stat().st_size, + } + ) + manifest = { + "ct_number": ct_number, + "selected_source_case_dir": path_text(candidate.source_case_dir), + "selected_source_stl_dir": path_text(candidate.source_stl_dir), + "processed_stl_dir": path_text(ct_dir), + "selection_rule": "file_count_desc_then_sequence_no_desc", + "stl_file_count": candidate.file_count, + "stl_total_bytes": candidate.total_bytes, + "files": file_rows, + } + (ct_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") + processed_files[ct_number] = file_rows + link_methods[ct_number] = "+".join(sorted(used_methods)) + return processed_files, link_methods + + +def load_list_records(list_json: Path) -> tuple[dict[str, list[dict[str, Any]]], list[dict[str, Any]]]: + data = json.loads(list_json.read_text(encoding="utf-8")) + records_by_ct: dict[str, list[dict[str, Any]]] = {} + invalid: list[dict[str, Any]] = [] + for index, record in enumerate(data.get("图片表格记录", []), start=1): + record = clean_record(record) + info = record.get("记录信息") or {} + ct_number = normalize_ct(info.get("检查号", "")) + if not valid_ct(ct_number): + invalid.append({"row_index": index, "ct_number": ct_number, "record": record}) + continue + records_by_ct.setdefault(ct_number, []).append(record) + return records_by_ct, invalid + + +def select_list_record(records: list[dict[str, Any]]) -> dict[str, Any]: + def key(record: dict[str, Any]) -> tuple[datetime, datetime, int, int]: + info = record.get("记录信息") or {} + image = record.get("图片信息") or {} + task_time = parse_time(info.get("任务创建时间")) or datetime.min + exam_time = parse_time(info.get("检查日期")) or datetime.min + page_no = -1 + seq = image.get("图片序号") + if isinstance(seq, list): + numbers = [item for item in seq if isinstance(item, int)] + if numbers: + page_no = numbers[0] + row_no = int(image.get("图片内行号") or -1) + return (task_time, exam_time, page_no, row_no) + + return max(records, key=key) + + +def build_rows( + selected: dict[str, StlCandidate], + all_candidates: dict[str, list[StlCandidate]], + processed_root: Path, + processed_files: dict[str, list[dict[str, Any]]], + list_records: dict[str, list[dict[str, Any]]], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], int]: + asset_rows: list[dict[str, Any]] = [] + stl_rows: list[dict[str, Any]] = [] + stl_file_count = 0 + for ct_number in sorted(set(selected) | set(list_records)): + candidate = selected.get(ct_number) + records = list_records.get(ct_number, []) + chosen_record = select_list_record(records) if records else None + info = (chosen_record or {}).get("记录信息") or {} + files = processed_files.get(ct_number, []) + processed_dir = processed_root / ct_number + asset_rows.append( + { + "ct_number": ct_number, + "list_present": bool(records), + "stl_present": candidate is not None, + "patient_name": normalize_text(info.get("姓名")), + "patient_sex": normalize_text(info.get("性别")), + "patient_age": normalize_text(info.get("年龄")), + "patient_id_masked": normalize_text(info.get("患者号")), + "exam_date": normalize_text(info.get("检查日期")), + "task_created_at": normalize_text(info.get("任务创建时间")), + "exam_description": normalize_text(info.get("检查描述")), + "exam_device": normalize_text(info.get("检查设备")), + "algorithm_model": normalize_text(info.get("算法模型")), + "upp_status": normalize_text(info.get("状态")), + "list_record_count": len(records), + "selected_list_record": json_dump(chosen_record) if chosen_record else "", + "list_records": json_dump(records), + "selected_source_case_dir": path_text(candidate.source_case_dir) if candidate else "", + "selected_source_stl_dir": path_text(candidate.source_stl_dir) if candidate else "", + "processed_stl_dir": path_text(processed_dir) if candidate else "", + "stl_case_name": candidate.source_case_dir.name if candidate else "", + "stl_sequence_no": candidate.sequence_no if candidate and candidate.sequence_no is not None else "", + "stl_file_count": candidate.file_count if candidate else 0, + "stl_total_bytes": candidate.total_bytes if candidate else 0, + "stl_files": json_dump(files), + "stl_candidates": json_dump([item.summary() for item in sorted(all_candidates.get(ct_number, []), key=lambda item: item.score(), reverse=True)]), + } + ) + if files: + stl_file_count += len(files) + stl_rows.append( + { + "ct_number": ct_number, + "file_count": len(files), + "total_bytes": sum(int(file_info.get("size_bytes") or 0) for file_info in files), + "segment_names": json_dump([file_info["segment_name"] for file_info in files]), + "segment_families": json_dump([file_info["family"] for file_info in files]), + "segment_categories": json_dump([file_info["category"] for file_info in files]), + "file_names": json_dump([file_info["file_name"] for file_info in files]), + "source_file_paths": json_dump([file_info["source_file_path"] for file_info in files]), + "processed_file_paths": json_dump([file_info["processed_file_path"] for file_info in files]), + "files": json_dump(files), + } + ) + return asset_rows, stl_rows, stl_file_count + + +def write_csv(rows: list[dict[str, Any]], fields: list[str], suffix: str) -> Path: + temp_file = tempfile.NamedTemporaryFile("w", encoding="utf-8", newline="", suffix=suffix, delete=False) + with temp_file: + writer = csv.DictWriter(temp_file, fieldnames=fields) + writer.writeheader() + for row in rows: + writer.writerow({field: row.get(field, "") for field in fields}) + return Path(temp_file.name) + + +def run_psql(args: argparse.Namespace, asset_csv: Path, stl_csv: Path) -> None: + sql = f""" +\\set ON_ERROR_STOP on +\\i {sql_quote_path(Path(args.schema))} +CREATE TEMP TABLE stg_upp_exam_assets ( + ct_number text, + list_present boolean, + stl_present boolean, + patient_name text, + patient_sex text, + patient_age text, + patient_id_masked text, + exam_date timestamptz, + task_created_at timestamptz, + exam_description text, + exam_device text, + algorithm_model text, + upp_status text, + list_record_count integer, + selected_list_record text, + list_records text, + selected_source_case_dir text, + selected_source_stl_dir text, + processed_stl_dir text, + stl_case_name text, + stl_sequence_no integer, + stl_file_count integer, + stl_total_bytes bigint, + stl_files text, + stl_candidates text +); +CREATE TEMP TABLE stg_upp_stl_files ( + ct_number text, + file_count integer, + total_bytes bigint, + segment_names text, + segment_families text, + segment_categories text, + file_names text, + source_file_paths text, + processed_file_paths text, + files text +); +\\copy stg_upp_exam_assets({",".join(ASSET_FIELDS)}) FROM {sql_quote_path(asset_csv)} WITH (FORMAT csv, HEADER true, NULL '') +\\copy stg_upp_stl_files({",".join(STL_FIELDS)}) FROM {sql_quote_path(stl_csv)} WITH (FORMAT csv, HEADER true, NULL '') +INSERT INTO upp_exam_assets ( + ct_number, list_present, stl_present, patient_name, patient_sex, patient_age, patient_id_masked, + exam_date, task_created_at, exam_description, exam_device, algorithm_model, upp_status, + list_record_count, selected_list_record, list_records, selected_source_case_dir, selected_source_stl_dir, + processed_stl_dir, stl_case_name, stl_sequence_no, stl_file_count, stl_total_bytes, stl_files, stl_candidates, updated_at +) +SELECT + ct_number, list_present, stl_present, patient_name, patient_sex, patient_age, patient_id_masked, + exam_date, task_created_at, exam_description, exam_device, algorithm_model, upp_status, + COALESCE(list_record_count, 0), + NULLIF(selected_list_record, '')::jsonb, + COALESCE(NULLIF(list_records, ''), '[]')::jsonb, + selected_source_case_dir, selected_source_stl_dir, processed_stl_dir, stl_case_name, stl_sequence_no, + COALESCE(stl_file_count, 0), COALESCE(stl_total_bytes, 0), + COALESCE(NULLIF(stl_files, ''), '[]')::jsonb, + COALESCE(NULLIF(stl_candidates, ''), '[]')::jsonb, + now() +FROM stg_upp_exam_assets +ON CONFLICT (ct_number) DO UPDATE SET + list_present = EXCLUDED.list_present, + stl_present = EXCLUDED.stl_present, + patient_name = EXCLUDED.patient_name, + patient_sex = EXCLUDED.patient_sex, + patient_age = EXCLUDED.patient_age, + patient_id_masked = EXCLUDED.patient_id_masked, + exam_date = EXCLUDED.exam_date, + task_created_at = EXCLUDED.task_created_at, + exam_description = EXCLUDED.exam_description, + exam_device = EXCLUDED.exam_device, + algorithm_model = EXCLUDED.algorithm_model, + upp_status = EXCLUDED.upp_status, + list_record_count = EXCLUDED.list_record_count, + selected_list_record = EXCLUDED.selected_list_record, + list_records = EXCLUDED.list_records, + selected_source_case_dir = EXCLUDED.selected_source_case_dir, + selected_source_stl_dir = EXCLUDED.selected_source_stl_dir, + processed_stl_dir = EXCLUDED.processed_stl_dir, + stl_case_name = EXCLUDED.stl_case_name, + stl_sequence_no = EXCLUDED.stl_sequence_no, + stl_file_count = EXCLUDED.stl_file_count, + stl_total_bytes = EXCLUDED.stl_total_bytes, + stl_files = EXCLUDED.stl_files, + stl_candidates = EXCLUDED.stl_candidates, + updated_at = now(); +DELETE FROM upp_stl_files WHERE ct_number IN (SELECT ct_number FROM stg_upp_exam_assets); +INSERT INTO upp_stl_files ( + ct_number, file_count, total_bytes, segment_names, segment_families, segment_categories, file_names, source_file_paths, processed_file_paths, files, updated_at +) +SELECT + ct_number, + COALESCE(file_count, 0), + COALESCE(total_bytes, 0), + COALESCE(NULLIF(segment_names, ''), '[]')::jsonb, + COALESCE(NULLIF(segment_families, ''), '[]')::jsonb, + COALESCE(NULLIF(segment_categories, ''), '[]')::jsonb, + COALESCE(NULLIF(file_names, ''), '[]')::jsonb, + COALESCE(NULLIF(source_file_paths, ''), '[]')::jsonb, + COALESCE(NULLIF(processed_file_paths, ''), '[]')::jsonb, + COALESCE(NULLIF(files, ''), '[]')::jsonb, + now() +FROM stg_upp_stl_files; +""" + env = os.environ.copy() + password = args.password or env.get("PGPASSWORD") + if password: + env["PGPASSWORD"] = password + command = ["psql"] + if args.host: + command.extend(["-h", args.host]) + if args.port: + command.extend(["-p", str(args.port)]) + if args.user: + command.extend(["-U", args.user]) + if args.dbname: + command.extend(["-d", args.dbname]) + command.extend(["-v", "ON_ERROR_STOP=1"]) + completed = subprocess.run(command, input=sql, text=True, env=env, capture_output=True) + print(completed.stdout, end="") + if completed.returncode != 0: + print(completed.stderr, end="") + completed.check_returncode() + + +def write_report(path: Path, report: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--stl-root", default=str(DEFAULT_STL_ROOT), help="待处理STL数据根目录") + parser.add_argument("--processed-root", default=str(DEFAULT_PROCESSED_ROOT), help="规范化后的STL输出目录") + parser.add_argument("--list-json", default=str(DEFAULT_LIST_JSON), help="UPP列表合并结构化JSON") + parser.add_argument("--schema", default=str(DEFAULT_SCHEMA), help="PostgreSQL建表SQL") + parser.add_argument("--report", default=str(DEFAULT_REPORT), help="同步报告JSON") + parser.add_argument("--link-mode", choices=["hardlink", "copy", "symlink"], default="hardlink", help="输出STL文件方式") + parser.add_argument("--no-refresh-files", action="store_true", help="不清空已处理CT目录中的旧文件") + parser.add_argument("--dry-run", action="store_true", help="只整理文件和生成报告，不写数据库") + parser.add_argument("--host", default=os.getenv("PGHOST", "")) + parser.add_argument("--port", default=os.getenv("PGPORT", "")) + parser.add_argument("--dbname", default=os.getenv("PGDATABASE", "")) + parser.add_argument("--user", default=os.getenv("PGUSER", "")) + parser.add_argument("--password", default=os.getenv("PGPASSWORD", ""), help="数据库密码；也可用PGPASSWORD环境变量") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + stl_root = Path(args.stl_root) + processed_root = Path(args.processed_root) + all_candidates, no_ct_dirs = scan_stl_candidates(stl_root) + selected = choose_candidates(all_candidates) + processed_files, link_methods = materialize_selected_stl( + selected=selected, + processed_root=processed_root, + link_mode=args.link_mode, + refresh_files=not args.no_refresh_files, + ) + list_records, invalid_list_records = load_list_records(Path(args.list_json)) + asset_rows, stl_rows, stl_file_count = build_rows( + selected=selected, + all_candidates=all_candidates, + processed_root=processed_root, + processed_files=processed_files, + list_records=list_records, + ) + duplicate_ct_numbers = sorted(ct_number for ct_number, items in all_candidates.items() if len(items) > 1) + report = { + "stl_candidate_dirs": sum(len(items) for items in all_candidates.values()), + "stl_unique_ct": len(selected), + "stl_no_ct_dirs": len(no_ct_dirs), + "stl_duplicate_ct": len(duplicate_ct_numbers), + "list_unique_valid_ct": len(list_records), + "list_invalid_ct_records": len(invalid_list_records), + "asset_rows": len(asset_rows), + "stl_table_rows": len(stl_rows), + "stl_file_rows": stl_file_count, + "matched_list_and_stl_ct": len(set(selected) & set(list_records)), + "list_without_stl_ct": len(set(list_records) - set(selected)), + "stl_without_list_ct": len(set(selected) - set(list_records)), + "duplicate_ct_examples": { + ct_number: [item.summary() for item in sorted(all_candidates[ct_number], key=lambda item: item.score(), reverse=True)] + for ct_number in duplicate_ct_numbers[:20] + }, + "no_ct_dir_examples": no_ct_dirs[:20], + "invalid_list_ct_examples": invalid_list_records[:20], + "link_methods": dict(sorted(link_methods.items())[:20]), + "processed_root": path_text(processed_root), + "selection_rule": "同一CT号优先选择STL文件数更多；文件数相同选择目录/病例名中的较大序号", + "dry_run": args.dry_run, + } + write_report(Path(args.report), report) + asset_csv = write_csv(asset_rows, ASSET_FIELDS, "_upp_exam_assets.csv") + stl_csv = write_csv(stl_rows, STL_FIELDS, "_upp_stl_files.csv") + try: + if not args.dry_run: + run_psql(args, asset_csv, stl_csv) + finally: + asset_csv.unlink(missing_ok=True) + stl_csv.unlink(missing_ok=True) + print(json.dumps({k: report[k] for k in [ + "stl_candidate_dirs", + "stl_unique_ct", + "stl_no_ct_dirs", + "stl_duplicate_ct", + "list_unique_valid_ct", + "asset_rows", + "stl_table_rows", + "stl_file_rows", + "matched_list_and_stl_ct", + "list_without_stl_ct", + "stl_without_list_ct", + ]}, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/UPP_数据库构建/03_统计STL名称.py b/UPP_数据库构建/03_统计STL名称.py new file mode 100755 index 0000000..0797977 --- /dev/null +++ b/UPP_数据库构建/03_统计STL名称.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""统计已处理STL名称，并按可解释的医学/结构类别归类。""" + +from __future__ import annotations + +import csv +import json +import re +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any + + +BASE_DIR = Path(__file__).resolve().parents[1] +DEFAULT_PROCESSED_ROOT = BASE_DIR / "UPP_STL处理" / "已处理STL数据" +DEFAULT_JSON = BASE_DIR / "UPP_数据库构建" / "UPP_STL名称统计.json" +DEFAULT_CSV = BASE_DIR / "UPP_数据库构建" / "UPP_STL名称统计.csv" +DEFAULT_FAMILY_CSV = BASE_DIR / "UPP_数据库构建" / "UPP_STL_family统计.csv" +DEFAULT_ORDERED_CSV = BASE_DIR / "UPP_数据库构建" / "UPP_STL文件family顺序明细.csv" + + +def classify(segment_name: str) -> tuple[str, str]: + if segment_name in {"liver", "liver_left", "liver_right"}: + return "肝脏主体", segment_name + if re.fullmatch(r"liver_segment_S[1-8]", segment_name): + return "肝段", segment_name + if segment_name in {"liver_artery", "liver_vein", "portal_vein", "bile_duct"}: + return "血管胆管", segment_name + if segment_name in {"pancreas", "spleen", "cholecyst"}: + return "腹部脏器", segment_name + if segment_name in {"skin", "rib", "vertebrae", "sternum", "hipbone", "sacrum"}: + return "体表骨骼", segment_name + if re.fullmatch(r"liver_tumor_\d+", segment_name): + return "肝脏肿瘤", "liver_tumor_*" + if re.fullmatch(r"liver_cyst_\d+", segment_name): + return "肝囊肿", "liver_cyst_*" + if re.fullmatch(r"liver_hemangioma_\d+", segment_name): + return "肝血管瘤", "liver_hemangioma_*" + if re.fullmatch(r"pancreas_tumor_\d+", segment_name): + return "胰腺肿瘤", "pancreas_tumor_*" + if re.fullmatch(r"Segment_\d+", segment_name): + return "未命名分割", "Segment_*" + return "其他", segment_name + + +def collect(processed_root: Path) -> dict[str, Any]: + segment_counter: Counter[str] = Counter() + category_counter: Counter[str] = Counter() + family_counter: Counter[str] = Counter() + family_ct: dict[str, set[str]] = defaultdict(set) + category_ct: dict[str, set[str]] = defaultdict(set) + name_ct: dict[str, set[str]] = defaultdict(set) + family_members: dict[str, set[str]] = defaultdict(set) + ordered_files: list[dict[str, Any]] = [] + ct_dirs = [item for item in processed_root.iterdir() if item.is_dir()] + + for ct_dir in sorted(ct_dirs): + ct_number = ct_dir.name + for order_no, stl_file in enumerate(sorted(ct_dir.glob("*.stl")), start=1): + segment_name = stl_file.stem + category, family = classify(segment_name) + segment_counter[segment_name] += 1 + category_counter[category] += 1 + family_counter[family] += 1 + name_ct[segment_name].add(ct_number) + family_ct[family].add(ct_number) + category_ct[category].add(ct_number) + family_members[family].add(segment_name) + ordered_files.append( + { + "ct_number": ct_number, + "order_no": order_no, + "segment_name": segment_name, + "family": family, + "category": category, + "file_name": stl_file.name, + "file_path": str(stl_file), + } + ) + + by_name = [ + { + "segment_name": name, + "category": classify(name)[0], + "family": classify(name)[1], + "file_count": count, + "ct_count": len(name_ct[name]), + } + for name, count in segment_counter.most_common() + ] + by_category = [ + { + "category": category, + "file_count": count, + "ct_count": len(category_ct[category]), + } + for category, count in category_counter.most_common() + ] + by_family = [ + { + "family": family, + "category": classify(family.replace("*", "1"))[0] if "*" in family else classify(family)[0], + "file_count": count, + "ct_count": len(family_ct[family]), + "segment_name_count": len(family_members[family]), + "segment_names": "|".join(sorted(family_members[family])), + } + for family, count in family_counter.most_common() + ] + return { + "processed_root": str(processed_root.relative_to(BASE_DIR)), + "ct_count": len(ct_dirs), + "stl_file_count": sum(segment_counter.values()), + "unique_segment_name_count": len(segment_counter), + "by_category": by_category, + "by_family": by_family, + "by_name": by_name, + "ordered_files": ordered_files, + } + + +def write_csv(path: Path, rows: list[dict[str, Any]], fieldnames: list[str]) -> None: + with path.open("w", encoding="utf-8", newline="") as file: + writer = csv.DictWriter(file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def main() -> None: + result = collect(DEFAULT_PROCESSED_ROOT) + ordered_files = result.pop("ordered_files") + DEFAULT_JSON.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + write_csv(DEFAULT_CSV, result["by_name"], ["segment_name", "category", "family", "file_count", "ct_count"]) + write_csv( + DEFAULT_FAMILY_CSV, + result["by_family"], + ["family", "category", "file_count", "ct_count", "segment_name_count", "segment_names"], + ) + write_csv( + DEFAULT_ORDERED_CSV, + ordered_files, + ["ct_number", "order_no", "segment_name", "family", "category", "file_name", "file_path"], + ) + print(json.dumps({ + "ct_count": result["ct_count"], + "stl_file_count": result["stl_file_count"], + "unique_segment_name_count": result["unique_segment_name_count"], + "json": str(DEFAULT_JSON), + "csv": str(DEFAULT_CSV), + "family_csv": str(DEFAULT_FAMILY_CSV), + "ordered_csv": str(DEFAULT_ORDERED_CSV), + }, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/UPP_数据库构建/README.md b/UPP_数据库构建/README.md new file mode 100644 index 0000000..0327efa --- /dev/null +++ b/UPP_数据库构建/README.md @@ -0,0 +1,45 @@ +# UPP STL资产索引 + +本目录用于把 `UPP_STL处理/待处理STL数据` 和 `UPP列表处理` 的合并 list 建成 CT 号唯一索引。 + +处理规则： + +- STL 候选目录必须能从路径中提取 `CT号`，格式为 `CT...` 或 `DCT...`。 +- 同一 CT 号有多个 STL 目录时，优先选择 STL 文件数更多的目录。 +- 文件数相同，则选择病例名或 `*-STL` 目录中序号更大的目录。 +- 规范化输出到 `UPP_STL处理/已处理STL数据//`。 +- 数据库中 `upp_exam_assets.ct_number` 是主键，可用 list 的检查号查询对应 STL 路径；未来 CT 数据也可以继续挂在同一行。 + +运行： + +```bash +cd /home/wkmgc/Desktop/PACS数据处理 +PGHOST='' PGPORT='' PGDATABASE='' PGUSER='' PGPASSWORD='' \ + python3 UPP_数据库构建/02_同步UPP_STL资产.py +``` + +主要表： + +- `upp_exam_assets`：CT 号唯一资产索引，含 list 元数据、最终选中 STL 目录、候选目录、STL 文件清单。 +- `upp_stl_files`：最终选中 STL 的 CT 级聚合表，每个 CT 号唯一一行，文件名、分割名、family、分类、路径和完整文件清单用 JSONB 存储。 + +STL 名称统计： + +```bash +python3 UPP_数据库构建/03_统计STL名称.py +``` + +输出： + +- `UPP_数据库构建/UPP_STL名称统计.json` +- `UPP_数据库构建/UPP_STL名称统计.csv` +- `UPP_数据库构建/UPP_STL_family统计.csv` +- `UPP_数据库构建/UPP_STL文件family顺序明细.csv`，含逐 CT 文件路径，默认不提交 + +常用查询： + +```sql +SELECT ct_number, patient_name, processed_stl_dir, stl_file_count +FROM upp_exam_assets +WHERE ct_number = ''; +``` diff --git a/UPP_数据库构建/UPP_STL_family统计.csv b/UPP_数据库构建/UPP_STL_family统计.csv new file mode 100644 index 0000000..4e8bbfe --- /dev/null +++ b/UPP_数据库构建/UPP_STL_family统计.csv @@ -0,0 +1,30 @@ +family,category,file_count,ct_count,segment_name_count,segment_names +liver_cyst_*,肝囊肿,945,272,37,liver_cyst_1|liver_cyst_10|liver_cyst_11|liver_cyst_12|liver_cyst_13|liver_cyst_14|liver_cyst_15|liver_cyst_16|liver_cyst_17|liver_cyst_18|liver_cyst_19|liver_cyst_2|liver_cyst_20|liver_cyst_21|liver_cyst_22|liver_cyst_23|liver_cyst_24|liver_cyst_25|liver_cyst_26|liver_cyst_27|liver_cyst_28|liver_cyst_29|liver_cyst_3|liver_cyst_30|liver_cyst_31|liver_cyst_32|liver_cyst_33|liver_cyst_34|liver_cyst_35|liver_cyst_36|liver_cyst_37|liver_cyst_4|liver_cyst_5|liver_cyst_6|liver_cyst_7|liver_cyst_8|liver_cyst_9 +liver_tumor_*,肝脏肿瘤,839,576,12,liver_tumor_1|liver_tumor_10|liver_tumor_11|liver_tumor_12|liver_tumor_2|liver_tumor_3|liver_tumor_4|liver_tumor_5|liver_tumor_6|liver_tumor_7|liver_tumor_8|liver_tumor_9 +liver,肝脏主体,725,725,1,liver +liver_left,肝脏主体,725,725,1,liver_left +liver_right,肝脏主体,725,725,1,liver_right +liver_segment_S1,肝段,725,725,1,liver_segment_S1 +liver_segment_S2,肝段,725,725,1,liver_segment_S2 +liver_segment_S3,肝段,725,725,1,liver_segment_S3 +liver_segment_S4,肝段,725,725,1,liver_segment_S4 +liver_segment_S5,肝段,725,725,1,liver_segment_S5 +liver_segment_S6,肝段,725,725,1,liver_segment_S6 +liver_segment_S7,肝段,725,725,1,liver_segment_S7 +liver_segment_S8,肝段,725,725,1,liver_segment_S8 +skin,体表骨骼,725,725,1,skin +liver_artery,血管胆管,724,724,1,liver_artery +portal_vein,血管胆管,724,724,1,portal_vein +rib,体表骨骼,724,724,1,rib +vertebrae,体表骨骼,724,724,1,vertebrae +liver_vein,血管胆管,723,723,1,liver_vein +pancreas,腹部脏器,723,723,1,pancreas +bile_duct,血管胆管,720,720,1,bile_duct +spleen,腹部脏器,717,717,1,spleen +sternum,体表骨骼,684,684,1,sternum +cholecyst,腹部脏器,565,565,1,cholecyst +hipbone,体表骨骼,455,455,1,hipbone +sacrum,体表骨骼,445,445,1,sacrum +liver_hemangioma_*,肝血管瘤,74,43,5,liver_hemangioma_1|liver_hemangioma_2|liver_hemangioma_3|liver_hemangioma_4|liver_hemangioma_5 +pancreas_tumor_*,胰腺肿瘤,12,12,1,pancreas_tumor_1 +Segment_*,未命名分割,9,5,4,Segment_0|Segment_1|Segment_2|Segment_3 diff --git a/UPP_数据库构建/UPP_STL名称统计.csv b/UPP_数据库构建/UPP_STL名称统计.csv new file mode 100644 index 0000000..8db6296 --- /dev/null +++ b/UPP_数据库构建/UPP_STL名称统计.csv @@ -0,0 +1,84 @@ +segment_name,category,family,file_count,ct_count +liver,肝脏主体,liver,725,725 +liver_left,肝脏主体,liver_left,725,725 +liver_right,肝脏主体,liver_right,725,725 +liver_segment_S1,肝段,liver_segment_S1,725,725 +liver_segment_S2,肝段,liver_segment_S2,725,725 +liver_segment_S3,肝段,liver_segment_S3,725,725 +liver_segment_S4,肝段,liver_segment_S4,725,725 +liver_segment_S5,肝段,liver_segment_S5,725,725 +liver_segment_S6,肝段,liver_segment_S6,725,725 +liver_segment_S7,肝段,liver_segment_S7,725,725 +liver_segment_S8,肝段,liver_segment_S8,725,725 +skin,体表骨骼,skin,725,725 +liver_artery,血管胆管,liver_artery,724,724 +portal_vein,血管胆管,portal_vein,724,724 +rib,体表骨骼,rib,724,724 +vertebrae,体表骨骼,vertebrae,724,724 +liver_vein,血管胆管,liver_vein,723,723 +pancreas,腹部脏器,pancreas,723,723 +bile_duct,血管胆管,bile_duct,720,720 +spleen,腹部脏器,spleen,717,717 +sternum,体表骨骼,sternum,684,684 +liver_tumor_1,肝脏肿瘤,liver_tumor_*,576,576 +cholecyst,腹部脏器,cholecyst,565,565 +hipbone,体表骨骼,hipbone,455,455 +sacrum,体表骨骼,sacrum,445,445 +liver_cyst_1,肝囊肿,liver_cyst_*,272,272 +liver_cyst_2,肝囊肿,liver_cyst_*,154,154 +liver_tumor_2,肝脏肿瘤,liver_tumor_*,143,143 +liver_cyst_3,肝囊肿,liver_cyst_*,102,102 +liver_cyst_4,肝囊肿,liver_cyst_*,70,70 +liver_cyst_5,肝囊肿,liver_cyst_*,59,59 +liver_tumor_3,肝脏肿瘤,liver_tumor_*,50,50 +liver_cyst_6,肝囊肿,liver_cyst_*,49,49 +liver_hemangioma_1,肝血管瘤,liver_hemangioma_*,43,43 +liver_cyst_7,肝囊肿,liver_cyst_*,36,36 +liver_cyst_8,肝囊肿,liver_cyst_*,28,28 +liver_tumor_4,肝脏肿瘤,liver_tumor_*,25,25 +liver_cyst_9,肝囊肿,liver_cyst_*,23,23 +liver_cyst_10,肝囊肿,liver_cyst_*,20,20 +liver_hemangioma_2,肝血管瘤,liver_hemangioma_*,16,16 +liver_cyst_11,肝囊肿,liver_cyst_*,15,15 +liver_cyst_12,肝囊肿,liver_cyst_*,12,12 +liver_cyst_13,肝囊肿,liver_cyst_*,12,12 +pancreas_tumor_1,胰腺肿瘤,pancreas_tumor_*,12,12 +liver_tumor_5,肝脏肿瘤,liver_tumor_*,12,12 +liver_cyst_14,肝囊肿,liver_cyst_*,11,11 +liver_tumor_6,肝脏肿瘤,liver_tumor_*,10,10 +liver_cyst_15,肝囊肿,liver_cyst_*,9,9 +liver_hemangioma_3,肝血管瘤,liver_hemangioma_*,9,9 +liver_tumor_7,肝脏肿瘤,liver_tumor_*,9,9 +liver_cyst_16,肝囊肿,liver_cyst_*,8,8 +liver_cyst_17,肝囊肿,liver_cyst_*,7,7 +liver_cyst_18,肝囊肿,liver_cyst_*,6,6 +liver_cyst_19,肝囊肿,liver_cyst_*,6,6 +liver_cyst_20,肝囊肿,liver_cyst_*,6,6 +liver_cyst_21,肝囊肿,liver_cyst_*,5,5 +liver_tumor_8,肝脏肿瘤,liver_tumor_*,5,5 +liver_hemangioma_4,肝血管瘤,liver_hemangioma_*,4,4 +liver_cyst_22,肝囊肿,liver_cyst_*,4,4 +liver_cyst_23,肝囊肿,liver_cyst_*,4,4 +liver_cyst_24,肝囊肿,liver_cyst_*,4,4 +liver_cyst_25,肝囊肿,liver_cyst_*,4,4 +Segment_0,未命名分割,Segment_*,4,4 +liver_tumor_9,肝脏肿瘤,liver_tumor_*,4,4 +liver_cyst_26,肝囊肿,liver_cyst_*,3,3 +liver_cyst_27,肝囊肿,liver_cyst_*,3,3 +liver_cyst_28,肝囊肿,liver_cyst_*,3,3 +liver_tumor_10,肝脏肿瘤,liver_tumor_*,3,3 +liver_hemangioma_5,肝血管瘤,liver_hemangioma_*,2,2 +liver_cyst_29,肝囊肿,liver_cyst_*,2,2 +Segment_1,未命名分割,Segment_*,2,2 +Segment_2,未命名分割,Segment_*,2,2 +liver_cyst_30,肝囊肿,liver_cyst_*,1,1 +liver_cyst_31,肝囊肿,liver_cyst_*,1,1 +liver_cyst_32,肝囊肿,liver_cyst_*,1,1 +liver_cyst_33,肝囊肿,liver_cyst_*,1,1 +liver_cyst_34,肝囊肿,liver_cyst_*,1,1 +liver_cyst_35,肝囊肿,liver_cyst_*,1,1 +liver_cyst_36,肝囊肿,liver_cyst_*,1,1 +liver_cyst_37,肝囊肿,liver_cyst_*,1,1 +Segment_3,未命名分割,Segment_*,1,1 +liver_tumor_11,肝脏肿瘤,liver_tumor_*,1,1 +liver_tumor_12,肝脏肿瘤,liver_tumor_*,1,1 diff --git a/UPP_数据库构建/UPP_STL名称统计.json b/UPP_数据库构建/UPP_STL名称统计.json new file mode 100644 index 0000000..2a8fecc --- /dev/null +++ b/UPP_数据库构建/UPP_STL名称统计.json @@ -0,0 +1,875 @@ +{ + "processed_root": "UPP_STL处理/已处理STL数据", + "ct_count": 726, + "stl_file_count": 18507, + "unique_segment_name_count": 83, + "by_category": [ + { + "category": "肝段", + "file_count": 5800, + "ct_count": 725 + }, + { + "category": "体表骨骼", + "file_count": 3757, + "ct_count": 726 + }, + { + "category": "血管胆管", + "file_count": 2891, + "ct_count": 724 + }, + { + "category": "肝脏主体", + "file_count": 2175, + "ct_count": 725 + }, + { + "category": "腹部脏器", + "file_count": 2005, + "ct_count": 724 + }, + { + "category": "肝囊肿", + "file_count": 945, + "ct_count": 272 + }, + { + "category": "肝脏肿瘤", + "file_count": 839, + "ct_count": 576 + }, + { + "category": "肝血管瘤", + "file_count": 74, + "ct_count": 43 + }, + { + "category": "胰腺肿瘤", + "file_count": 12, + "ct_count": 12 + }, + { + "category": "未命名分割", + "file_count": 9, + "ct_count": 5 + } + ], + "by_family": [ + { + "family": "liver_cyst_*", + "category": "肝囊肿", + "file_count": 945, + "ct_count": 272, + "segment_name_count": 37, + "segment_names": "liver_cyst_1|liver_cyst_10|liver_cyst_11|liver_cyst_12|liver_cyst_13|liver_cyst_14|liver_cyst_15|liver_cyst_16|liver_cyst_17|liver_cyst_18|liver_cyst_19|liver_cyst_2|liver_cyst_20|liver_cyst_21|liver_cyst_22|liver_cyst_23|liver_cyst_24|liver_cyst_25|liver_cyst_26|liver_cyst_27|liver_cyst_28|liver_cyst_29|liver_cyst_3|liver_cyst_30|liver_cyst_31|liver_cyst_32|liver_cyst_33|liver_cyst_34|liver_cyst_35|liver_cyst_36|liver_cyst_37|liver_cyst_4|liver_cyst_5|liver_cyst_6|liver_cyst_7|liver_cyst_8|liver_cyst_9" + }, + { + "family": "liver_tumor_*", + "category": "肝脏肿瘤", + "file_count": 839, + "ct_count": 576, + "segment_name_count": 12, + "segment_names": "liver_tumor_1|liver_tumor_10|liver_tumor_11|liver_tumor_12|liver_tumor_2|liver_tumor_3|liver_tumor_4|liver_tumor_5|liver_tumor_6|liver_tumor_7|liver_tumor_8|liver_tumor_9" + }, + { + "family": "liver", + "category": "肝脏主体", + "file_count": 725, + "ct_count": 725, + "segment_name_count": 1, + "segment_names": "liver" + }, + { + "family": "liver_left", + "category": "肝脏主体", + "file_count": 725, + "ct_count": 725, + "segment_name_count": 1, + "segment_names": "liver_left" + }, + { + "family": "liver_right", + "category": "肝脏主体", + "file_count": 725, + "ct_count": 725, + "segment_name_count": 1, + "segment_names": "liver_right" + }, + { + "family": "liver_segment_S1", + "category": "肝段", + "file_count": 725, + "ct_count": 725, + "segment_name_count": 1, + "segment_names": "liver_segment_S1" + }, + { + "family": "liver_segment_S2", + "category": "肝段", + "file_count": 725, + "ct_count": 725, + "segment_name_count": 1, + "segment_names": "liver_segment_S2" + }, + { + "family": "liver_segment_S3", + "category": "肝段", + "file_count": 725, + "ct_count": 725, + "segment_name_count": 1, + "segment_names": "liver_segment_S3" + }, + { + "family": "liver_segment_S4", + "category": "肝段", + "file_count": 725, + "ct_count": 725, + "segment_name_count": 1, + "segment_names": "liver_segment_S4" + }, + { + "family": "liver_segment_S5", + "category": "肝段", + "file_count": 725, + "ct_count": 725, + "segment_name_count": 1, + "segment_names": "liver_segment_S5" + }, + { + "family": "liver_segment_S6", + "category": "肝段", + "file_count": 725, + "ct_count": 725, + "segment_name_count": 1, + "segment_names": "liver_segment_S6" + }, + { + "family": "liver_segment_S7", + "category": "肝段", + "file_count": 725, + "ct_count": 725, + "segment_name_count": 1, + "segment_names": "liver_segment_S7" + }, + { + "family": "liver_segment_S8", + "category": "肝段", + "file_count": 725, + "ct_count": 725, + "segment_name_count": 1, + "segment_names": "liver_segment_S8" + }, + { + "family": "skin", + "category": "体表骨骼", + "file_count": 725, + "ct_count": 725, + "segment_name_count": 1, + "segment_names": "skin" + }, + { + "family": "liver_artery", + "category": "血管胆管", + "file_count": 724, + "ct_count": 724, + "segment_name_count": 1, + "segment_names": "liver_artery" + }, + { + "family": "portal_vein", + "category": "血管胆管", + "file_count": 724, + "ct_count": 724, + "segment_name_count": 1, + "segment_names": "portal_vein" + }, + { + "family": "rib", + "category": "体表骨骼", + "file_count": 724, + "ct_count": 724, + "segment_name_count": 1, + "segment_names": "rib" + }, + { + "family": "vertebrae", + "category": "体表骨骼", + "file_count": 724, + "ct_count": 724, + "segment_name_count": 1, + "segment_names": "vertebrae" + }, + { + "family": "liver_vein", + "category": "血管胆管", + "file_count": 723, + "ct_count": 723, + "segment_name_count": 1, + "segment_names": "liver_vein" + }, + { + "family": "pancreas", + "category": "腹部脏器", + "file_count": 723, + "ct_count": 723, + "segment_name_count": 1, + "segment_names": "pancreas" + }, + { + "family": "bile_duct", + "category": "血管胆管", + "file_count": 720, + "ct_count": 720, + "segment_name_count": 1, + "segment_names": "bile_duct" + }, + { + "family": "spleen", + "category": "腹部脏器", + "file_count": 717, + "ct_count": 717, + "segment_name_count": 1, + "segment_names": "spleen" + }, + { + "family": "sternum", + "category": "体表骨骼", + "file_count": 684, + "ct_count": 684, + "segment_name_count": 1, + "segment_names": "sternum" + }, + { + "family": "cholecyst", + "category": "腹部脏器", + "file_count": 565, + "ct_count": 565, + "segment_name_count": 1, + "segment_names": "cholecyst" + }, + { + "family": "hipbone", + "category": "体表骨骼", + "file_count": 455, + "ct_count": 455, + "segment_name_count": 1, + "segment_names": "hipbone" + }, + { + "family": "sacrum", + "category": "体表骨骼", + "file_count": 445, + "ct_count": 445, + "segment_name_count": 1, + "segment_names": "sacrum" + }, + { + "family": "liver_hemangioma_*", + "category": "肝血管瘤", + "file_count": 74, + "ct_count": 43, + "segment_name_count": 5, + "segment_names": "liver_hemangioma_1|liver_hemangioma_2|liver_hemangioma_3|liver_hemangioma_4|liver_hemangioma_5" + }, + { + "family": "pancreas_tumor_*", + "category": "胰腺肿瘤", + "file_count": 12, + "ct_count": 12, + "segment_name_count": 1, + "segment_names": "pancreas_tumor_1" + }, + { + "family": "Segment_*", + "category": "未命名分割", + "file_count": 9, + "ct_count": 5, + "segment_name_count": 4, + "segment_names": "Segment_0|Segment_1|Segment_2|Segment_3" + } + ], + "by_name": [ + { + "segment_name": "liver", + "category": "肝脏主体", + "family": "liver", + "file_count": 725, + "ct_count": 725 + }, + { + "segment_name": "liver_left", + "category": "肝脏主体", + "family": "liver_left", + "file_count": 725, + "ct_count": 725 + }, + { + "segment_name": "liver_right", + "category": "肝脏主体", + "family": "liver_right", + "file_count": 725, + "ct_count": 725 + }, + { + "segment_name": "liver_segment_S1", + "category": "肝段", + "family": "liver_segment_S1", + "file_count": 725, + "ct_count": 725 + }, + { + "segment_name": "liver_segment_S2", + "category": "肝段", + "family": "liver_segment_S2", + "file_count": 725, + "ct_count": 725 + }, + { + "segment_name": "liver_segment_S3", + "category": "肝段", + "family": "liver_segment_S3", + "file_count": 725, + "ct_count": 725 + }, + { + "segment_name": "liver_segment_S4", + "category": "肝段", + "family": "liver_segment_S4", + "file_count": 725, + "ct_count": 725 + }, + { + "segment_name": "liver_segment_S5", + "category": "肝段", + "family": "liver_segment_S5", + "file_count": 725, + "ct_count": 725 + }, + { + "segment_name": "liver_segment_S6", + "category": "肝段", + "family": "liver_segment_S6", + "file_count": 725, + "ct_count": 725 + }, + { + "segment_name": "liver_segment_S7", + "category": "肝段", + "family": "liver_segment_S7", + "file_count": 725, + "ct_count": 725 + }, + { + "segment_name": "liver_segment_S8", + "category": "肝段", + "family": "liver_segment_S8", + "file_count": 725, + "ct_count": 725 + }, + { + "segment_name": "skin", + "category": "体表骨骼", + "family": "skin", + "file_count": 725, + "ct_count": 725 + }, + { + "segment_name": "liver_artery", + "category": "血管胆管", + "family": "liver_artery", + "file_count": 724, + "ct_count": 724 + }, + { + "segment_name": "portal_vein", + "category": "血管胆管", + "family": "portal_vein", + "file_count": 724, + "ct_count": 724 + }, + { + "segment_name": "rib", + "category": "体表骨骼", + "family": "rib", + "file_count": 724, + "ct_count": 724 + }, + { + "segment_name": "vertebrae", + "category": "体表骨骼", + "family": "vertebrae", + "file_count": 724, + "ct_count": 724 + }, + { + "segment_name": "liver_vein", + "category": "血管胆管", + "family": "liver_vein", + "file_count": 723, + "ct_count": 723 + }, + { + "segment_name": "pancreas", + "category": "腹部脏器", + "family": "pancreas", + "file_count": 723, + "ct_count": 723 + }, + { + "segment_name": "bile_duct", + "category": "血管胆管", + "family": "bile_duct", + "file_count": 720, + "ct_count": 720 + }, + { + "segment_name": "spleen", + "category": "腹部脏器", + "family": "spleen", + "file_count": 717, + "ct_count": 717 + }, + { + "segment_name": "sternum", + "category": "体表骨骼", + "family": "sternum", + "file_count": 684, + "ct_count": 684 + }, + { + "segment_name": "liver_tumor_1", + "category": "肝脏肿瘤", + "family": "liver_tumor_*", + "file_count": 576, + "ct_count": 576 + }, + { + "segment_name": "cholecyst", + "category": "腹部脏器", + "family": "cholecyst", + "file_count": 565, + "ct_count": 565 + }, + { + "segment_name": "hipbone", + "category": "体表骨骼", + "family": "hipbone", + "file_count": 455, + "ct_count": 455 + }, + { + "segment_name": "sacrum", + "category": "体表骨骼", + "family": "sacrum", + "file_count": 445, + "ct_count": 445 + }, + { + "segment_name": "liver_cyst_1", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 272, + "ct_count": 272 + }, + { + "segment_name": "liver_cyst_2", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 154, + "ct_count": 154 + }, + { + "segment_name": "liver_tumor_2", + "category": "肝脏肿瘤", + "family": "liver_tumor_*", + "file_count": 143, + "ct_count": 143 + }, + { + "segment_name": "liver_cyst_3", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 102, + "ct_count": 102 + }, + { + "segment_name": "liver_cyst_4", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 70, + "ct_count": 70 + }, + { + "segment_name": "liver_cyst_5", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 59, + "ct_count": 59 + }, + { + "segment_name": "liver_tumor_3", + "category": "肝脏肿瘤", + "family": "liver_tumor_*", + "file_count": 50, + "ct_count": 50 + }, + { + "segment_name": "liver_cyst_6", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 49, + "ct_count": 49 + }, + { + "segment_name": "liver_hemangioma_1", + "category": "肝血管瘤", + "family": "liver_hemangioma_*", + "file_count": 43, + "ct_count": 43 + }, + { + "segment_name": "liver_cyst_7", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 36, + "ct_count": 36 + }, + { + "segment_name": "liver_cyst_8", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 28, + "ct_count": 28 + }, + { + "segment_name": "liver_tumor_4", + "category": "肝脏肿瘤", + "family": "liver_tumor_*", + "file_count": 25, + "ct_count": 25 + }, + { + "segment_name": "liver_cyst_9", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 23, + "ct_count": 23 + }, + { + "segment_name": "liver_cyst_10", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 20, + "ct_count": 20 + }, + { + "segment_name": "liver_hemangioma_2", + "category": "肝血管瘤", + "family": "liver_hemangioma_*", + "file_count": 16, + "ct_count": 16 + }, + { + "segment_name": "liver_cyst_11", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 15, + "ct_count": 15 + }, + { + "segment_name": "liver_cyst_12", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 12, + "ct_count": 12 + }, + { + "segment_name": "liver_cyst_13", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 12, + "ct_count": 12 + }, + { + "segment_name": "pancreas_tumor_1", + "category": "胰腺肿瘤", + "family": "pancreas_tumor_*", + "file_count": 12, + "ct_count": 12 + }, + { + "segment_name": "liver_tumor_5", + "category": "肝脏肿瘤", + "family": "liver_tumor_*", + "file_count": 12, + "ct_count": 12 + }, + { + "segment_name": "liver_cyst_14", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 11, + "ct_count": 11 + }, + { + "segment_name": "liver_tumor_6", + "category": "肝脏肿瘤", + "family": "liver_tumor_*", + "file_count": 10, + "ct_count": 10 + }, + { + "segment_name": "liver_cyst_15", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 9, + "ct_count": 9 + }, + { + "segment_name": "liver_hemangioma_3", + "category": "肝血管瘤", + "family": "liver_hemangioma_*", + "file_count": 9, + "ct_count": 9 + }, + { + "segment_name": "liver_tumor_7", + "category": "肝脏肿瘤", + "family": "liver_tumor_*", + "file_count": 9, + "ct_count": 9 + }, + { + "segment_name": "liver_cyst_16", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 8, + "ct_count": 8 + }, + { + "segment_name": "liver_cyst_17", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 7, + "ct_count": 7 + }, + { + "segment_name": "liver_cyst_18", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 6, + "ct_count": 6 + }, + { + "segment_name": "liver_cyst_19", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 6, + "ct_count": 6 + }, + { + "segment_name": "liver_cyst_20", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 6, + "ct_count": 6 + }, + { + "segment_name": "liver_cyst_21", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 5, + "ct_count": 5 + }, + { + "segment_name": "liver_tumor_8", + "category": "肝脏肿瘤", + "family": "liver_tumor_*", + "file_count": 5, + "ct_count": 5 + }, + { + "segment_name": "liver_hemangioma_4", + "category": "肝血管瘤", + "family": "liver_hemangioma_*", + "file_count": 4, + "ct_count": 4 + }, + { + "segment_name": "liver_cyst_22", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 4, + "ct_count": 4 + }, + { + "segment_name": "liver_cyst_23", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 4, + "ct_count": 4 + }, + { + "segment_name": "liver_cyst_24", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 4, + "ct_count": 4 + }, + { + "segment_name": "liver_cyst_25", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 4, + "ct_count": 4 + }, + { + "segment_name": "Segment_0", + "category": "未命名分割", + "family": "Segment_*", + "file_count": 4, + "ct_count": 4 + }, + { + "segment_name": "liver_tumor_9", + "category": "肝脏肿瘤", + "family": "liver_tumor_*", + "file_count": 4, + "ct_count": 4 + }, + { + "segment_name": "liver_cyst_26", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 3, + "ct_count": 3 + }, + { + "segment_name": "liver_cyst_27", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 3, + "ct_count": 3 + }, + { + "segment_name": "liver_cyst_28", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 3, + "ct_count": 3 + }, + { + "segment_name": "liver_tumor_10", + "category": "肝脏肿瘤", + "family": "liver_tumor_*", + "file_count": 3, + "ct_count": 3 + }, + { + "segment_name": "liver_hemangioma_5", + "category": "肝血管瘤", + "family": "liver_hemangioma_*", + "file_count": 2, + "ct_count": 2 + }, + { + "segment_name": "liver_cyst_29", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 2, + "ct_count": 2 + }, + { + "segment_name": "Segment_1", + "category": "未命名分割", + "family": "Segment_*", + "file_count": 2, + "ct_count": 2 + }, + { + "segment_name": "Segment_2", + "category": "未命名分割", + "family": "Segment_*", + "file_count": 2, + "ct_count": 2 + }, + { + "segment_name": "liver_cyst_30", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 1, + "ct_count": 1 + }, + { + "segment_name": "liver_cyst_31", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 1, + "ct_count": 1 + }, + { + "segment_name": "liver_cyst_32", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 1, + "ct_count": 1 + }, + { + "segment_name": "liver_cyst_33", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 1, + "ct_count": 1 + }, + { + "segment_name": "liver_cyst_34", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 1, + "ct_count": 1 + }, + { + "segment_name": "liver_cyst_35", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 1, + "ct_count": 1 + }, + { + "segment_name": "liver_cyst_36", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 1, + "ct_count": 1 + }, + { + "segment_name": "liver_cyst_37", + "category": "肝囊肿", + "family": "liver_cyst_*", + "file_count": 1, + "ct_count": 1 + }, + { + "segment_name": "Segment_3", + "category": "未命名分割", + "family": "Segment_*", + "file_count": 1, + "ct_count": 1 + }, + { + "segment_name": "liver_tumor_11", + "category": "肝脏肿瘤", + "family": "liver_tumor_*", + "file_count": 1, + "ct_count": 1 + }, + { + "segment_name": "liver_tumor_12", + "category": "肝脏肿瘤", + "family": "liver_tumor_*", + "file_count": 1, + "ct_count": 1 + } + ] +} \ No newline at end of file