Initial Seg Data Server Net platform

This commit is contained in:
2026-06-30 11:49:36 +08:00
commit 98abafa7cc
48 changed files with 6020 additions and 0 deletions

View File

@@ -0,0 +1,24 @@
from __future__ import annotations
from .analysis.tasks import build_analysis_task
from .dataset.tasks import build_dataset_task
from .mmseg.tasks import build_mmseg_task
from .segmodel.tasks import build_segmodel_task
from .system.tasks import build_system_task
from .yolo.tasks import build_yolo_task
def build_module_task(job_type: str, params: dict, conda_env: str):
for builder in (
build_dataset_task,
build_segmodel_task,
build_yolo_task,
build_mmseg_task,
build_analysis_task,
build_system_task,
):
spec = builder(job_type, params, conda_env)
if spec is not None:
return spec
return None

View File

@@ -0,0 +1,2 @@
"""Analysis task wrappers."""

View File

@@ -0,0 +1,18 @@
from __future__ import annotations
from ...commands import CommandSpec, append_flag, conda_python
from ...config import settings
ANALYSIS_DIR = settings.source_root / "Seg_All_In_One_Analysis"
def build_analysis_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
if job_type != "analysis.all":
return None
args = conda_python(conda_env, ANALYSIS_DIR / "1_Analysis_All.py")
append_flag(args, "--input_dir", params.get("input_dir", "../BestMode_Predict_Results_DataSet_Public"))
append_flag(args, "--output_dir", params.get("output_dir", "./"))
stdin = f"{params.get('dataset_choice', 1)}\n"
return CommandSpec(args, ANALYSIS_DIR, "merge SegModel/MMSeg metrics and generate plots", stdin_text=stdin)

View File

@@ -0,0 +1,2 @@
"""Dataset task wrappers."""

View File

@@ -0,0 +1,88 @@
from __future__ import annotations
from pathlib import Path
from ...commands import CommandSpec, append_flag, bash, conda_python, required
from ...config import settings
DATASET_TOOL_DIR = settings.source_root / "DataSet_Own" / "1. 图片预处理(内含使用手册)"
STACK_TOOL_DIR = settings.source_root / "Tool-图片堆叠"
VIDEO_DIR = settings.source_root / "Seg_Predict_Own_Video_V2"
YOLO_DATASET_DIR = settings.source_root / "Seg_All_In_One_YoloModel" / "Yolo数据集构建"
def _dataset_script(name: str) -> Path:
return DATASET_TOOL_DIR / name
def build_dataset_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
if job_type == "dataset.rename":
args = bash(_dataset_script("1_rename_pics.sh"))
append_flag(args, "-i", required(params, "image_dir"))
append_flag(args, "-l", required(params, "label_dir"))
return CommandSpec(args, DATASET_TOOL_DIR, "rename and normalize image/label names")
if job_type == "dataset.to_png":
script = _dataset_script("2_1_Trans_to_png.py")
args = conda_python(conda_env, script)
append_flag(args, "-i", params.get("input_dir"))
append_flag(args, "-o", params.get("output_dir"))
return CommandSpec(args, DATASET_TOOL_DIR, "convert images to png")
if job_type == "dataset.resize":
args = bash(_dataset_script("2_reformate_pics.sh"))
append_flag(args, "-i", params.get("image_dir"))
append_flag(args, "-l", params.get("label_dir"))
append_flag(args, "-w", params.get("width", 1920))
append_flag(args, "-h", params.get("height", 1080))
return CommandSpec(args, DATASET_TOOL_DIR, "resize and reformat image/label folders")
if job_type == "dataset.pair":
args = bash(_dataset_script("3_pair_ori_label.sh"))
append_flag(args, "-i", required(params, "image_dir"))
append_flag(args, "-l", required(params, "label_dir"))
append_flag(args, "-p", params.get("prefix", ""))
append_flag(args, "-s", params.get("suffix", ""))
return CommandSpec(args, DATASET_TOOL_DIR, "check image and label pairing")
if job_type == "dataset.rebuild_labels":
args = bash(_dataset_script("4_rebuild_labels.sh"))
append_flag(args, "-l", required(params, "label_dir"))
return CommandSpec(args, DATASET_TOOL_DIR, "rebuild color labels into GT masks")
if job_type == "dataset.stack":
args = bash(_dataset_script("5_TOOL_stack_pics.sh"))
append_flag(args, "-i", required(params, "image_dir"))
append_flag(args, "-l", required(params, "label_dir"))
append_flag(args, "-r", required(params, "result_dir"))
append_flag(args, "-a", params.get("alpha", 0.3))
append_flag(args, "-p", params.get("prefix", ""))
append_flag(args, "-s", params.get("suffix", ""))
return CommandSpec(args, DATASET_TOOL_DIR, "overlay image and label for inspection")
if job_type == "dataset.stitch":
args = bash(_dataset_script("6_TOOL_stitch_pics.sh"))
append_flag(args, "-i", required(params, "image_dir"))
append_flag(args, "-l", required(params, "label_dir"))
append_flag(args, "-r", required(params, "result_dir"))
return CommandSpec(args, DATASET_TOOL_DIR, "stitch image and label panels")
if job_type == "dataset.video_frames":
script = VIDEO_DIR / "1_Save_Frame_V2.py"
args = conda_python(conda_env, script)
append_flag(args, "--video", required(params, "video"))
append_flag(args, "--interval", params.get("interval", 0.5))
append_flag(args, "--resize", params.get("resize"))
append_flag(args, "--output_dir", params.get("output_dir"))
return CommandSpec(args, VIDEO_DIR, "extract video frames into DataSet_Public layout")
if job_type == "dataset.yolo_check_pairs":
script = YOLO_DATASET_DIR / "0_1_check_picture_pair.py"
args = conda_python(conda_env, script)
append_flag(args, "-i", required(params, "image_dir"))
append_flag(args, "-l", required(params, "label_dir"))
return CommandSpec(args, YOLO_DATASET_DIR, "check YOLO image/label pairs")
return None

View File

@@ -0,0 +1,2 @@
"""MMSeg task wrappers."""

View File

@@ -0,0 +1,94 @@
from __future__ import annotations
from ...commands import CommandSpec, append_flag, conda_python, required
from ...config import settings
MMSEG_DIR = settings.source_root / "Seg_All_In_One_MMSeg"
MY_DIR = MMSEG_DIR / "My_All_In_One"
def _stdin_for_generate_alg(params: dict) -> str:
lines = [
str(params.get("dataset_choice", 1)),
str(params.get("gpu_count", 1)),
]
gpu_ids = params.get("gpu_ids", [0])
if isinstance(gpu_ids, str):
gpu_ids = [part.strip() for part in gpu_ids.split(",") if part.strip()]
for index in range(int(params.get("gpu_count", len(gpu_ids) or 1))):
lines.append(str(gpu_ids[index] if index < len(gpu_ids) else 0))
mode = str(params.get("schedule_mode", 2))
lines.append(mode)
if mode == "1":
lines.extend(
[
str(params.get("train_k", 40)),
str(params.get("check_count", 10)),
str(params.get("logger_interval", 50)),
]
)
else:
lines.extend(
[
str(params.get("max_epochs", 300)),
str(params.get("val_interval", 1)),
str(params.get("checkpoint_interval", 10)),
str(params.get("logger_interval", "")),
]
)
lines.append(str(params.get("algorithm_choice", 1)))
return "\n".join(lines) + "\n"
def build_mmseg_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
if job_type == "mmseg.init_weights":
return CommandSpec(conda_python(conda_env, MY_DIR / "0_Initial_Save_All_Model_locally.py"), MMSEG_DIR, "download/save MMSeg pretrained weights locally")
if job_type == "mmseg.generate_data":
return CommandSpec(conda_python(conda_env, MY_DIR / "1_Initial_Data_All_data_from_1_Data_Parameter-V2.py"), MMSEG_DIR, "generate MMSeg dataset configs from JSON parameters")
if job_type == "mmseg.generate_alg":
script = MY_DIR / "2_Initial_Alg_All_data_from_2_Alg_Program-V2.py"
return CommandSpec(
conda_python(conda_env, script),
MMSEG_DIR,
"generate MMSeg algorithm config and training command",
stdin_text=_stdin_for_generate_alg(params),
)
if job_type == "mmseg.train":
config_path = required(params, "config")
args = conda_python(conda_env, MMSEG_DIR / "tools" / "train.py", config_path)
append_flag(args, "--work-dir", params.get("work_dir"))
return CommandSpec(args, MMSEG_DIR, "train MMSeg model")
if job_type == "mmseg.metrics":
args = conda_python(conda_env, MY_DIR / "4_2_predict_matrics_from_log_V2.py")
append_flag(args, "--input_dir", params.get("input_dir", "../Hardisk"))
append_flag(args, "--output_dir", params.get("output_dir", "../BestMode_Predict_Results_DataSet_Public"))
stdin = f"{params.get('dataset_choice', 1)}\n{params.get('algorithm_choice', 0)}\n"
return CommandSpec(args, MMSEG_DIR, "extract best MMSeg metrics from logs", stdin_text=stdin)
if job_type == "mmseg.flops_fps":
args = conda_python(conda_env, MY_DIR / "4_1_predict_params_FLOPs_FPS_V2.py")
append_flag(args, "--input_dir", params.get("input_dir", "../Hardisk"))
append_flag(args, "--output_dir", params.get("output_dir", "../BestMode_Predict_Results_DataSet_Public"))
append_flag(args, "--repeat-times", params.get("repeat_times", 3))
stdin = f"{params.get('dataset_choice', 1)}\n{params.get('algorithm_choice', 0)}\n"
if "shape_h" in params and "shape_w" in params:
stdin += f"{params['shape_h']}\n{params['shape_w']}\n"
return CommandSpec(args, MMSEG_DIR, "calculate MMSeg FLOPs/Params/FPS", stdin_text=stdin)
if job_type == "mmseg.draw":
return CommandSpec(conda_python(conda_env, MY_DIR / "4_3_predict_draw_pictures_and_tabels.py"), MMSEG_DIR, "generate MMSeg prediction pictures and tables")
if job_type == "mmseg.extract_loss_miou":
return CommandSpec(conda_python(conda_env, MY_DIR / "4_4_extract_loss_and_best_miou.py"), MMSEG_DIR, "extract MMSeg loss and best mIoU curves")
if job_type == "mmseg.delete_epoch":
return CommandSpec(conda_python(conda_env, MY_DIR / "3_Find_And_Delete_Special_Epoch.py"), MMSEG_DIR, "find and delete selected epoch checkpoints")
return None

View File

@@ -0,0 +1,2 @@
"""SegModel task wrappers."""

View File

@@ -0,0 +1,41 @@
from __future__ import annotations
from ...commands import CommandSpec, append_flag, bash, conda_python, required
from ...config import settings
SEGMODEL_DIR = settings.source_root / "Seg_All_In_One_SegModel"
def build_segmodel_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
env = {"SEG_CONDA_ENV": conda_env}
if job_type == "segmodel.train":
args = conda_python(conda_env, SEGMODEL_DIR / "train.py")
append_flag(args, "-a", required(params, "architecture"))
return CommandSpec(args, SEGMODEL_DIR, "train one segmentation_models_pytorch architecture")
if job_type == "segmodel.batch_train":
return CommandSpec(bash(SEGMODEL_DIR / "train.sh"), SEGMODEL_DIR, "run legacy SegModel batch training", env=env)
if job_type == "segmodel.predict":
args = conda_python(conda_env, SEGMODEL_DIR / "1_predict.py")
append_flag(args, "-a", required(params, "architecture"))
choice = str(params.get("run_choice", 1))
return CommandSpec(args, SEGMODEL_DIR, "predict with one SegModel run", stdin_text=f"{choice}\n")
if job_type == "segmodel.batch_predict":
return CommandSpec(bash(SEGMODEL_DIR / "predict.sh"), SEGMODEL_DIR, "run legacy SegModel batch prediction", env=env)
if job_type == "segmodel.flops":
script = SEGMODEL_DIR / params.get("script", "2_predict_params_and_FLOPs_V2.py")
return CommandSpec(conda_python(conda_env, script), SEGMODEL_DIR, "calculate SegModel params/FLOPs/FPS")
if job_type == "segmodel.raw_mask_check":
return CommandSpec(conda_python(conda_env, SEGMODEL_DIR / "1_predict_raw_masks_check.py"), SEGMODEL_DIR, "check SegModel raw mask completeness")
if job_type == "segmodel.metrics":
return CommandSpec(conda_python(conda_env, SEGMODEL_DIR / "3_predict_matrics_from_log.py"), SEGMODEL_DIR, "parse SegModel training/prediction metrics")
return None

View File

@@ -0,0 +1,2 @@
"""System task wrappers."""

View File

@@ -0,0 +1,106 @@
from __future__ import annotations
import shutil
import subprocess
from pathlib import Path
from ...config import settings
def parse_nvidia_smi_csv(output: str) -> list[dict]:
gpus: list[dict] = []
for line in output.splitlines():
if not line.strip():
continue
parts = [part.strip() for part in line.split(",")]
if len(parts) < 7:
continue
index, name, total, used, free, util, temp = parts[:7]
try:
gpus.append(
{
"index": int(index),
"name": name,
"memory_total_mb": int(total),
"memory_used_mb": int(used),
"memory_free_mb": int(free),
"utilization_gpu_percent": int(util),
"temperature_c": int(temp),
}
)
except ValueError:
continue
return gpus
def get_gpus() -> dict:
cmd = [
"nvidia-smi",
"--query-gpu=index,name,memory.total,memory.used,memory.free,utilization.gpu,temperature.gpu",
"--format=csv,noheader,nounits",
]
try:
result = subprocess.run(cmd, capture_output=True, text=True, check=True)
return {"available": True, "gpus": parse_nvidia_smi_csv(result.stdout)}
except Exception as exc:
return {"available": False, "gpus": [], "error": str(exc)}
def get_conda_envs() -> dict:
try:
result = subprocess.run(["conda", "env", "list"], capture_output=True, text=True, check=True)
except Exception as exc:
return {"available": False, "envs": [], "error": str(exc)}
envs = []
for line in result.stdout.splitlines():
raw = line.strip()
if not raw or raw.startswith("#"):
continue
marker = "*" in raw.split()
parts = raw.replace("*", " ").split()
if len(parts) >= 2:
envs.append({"name": parts[0], "path": parts[-1], "active": marker})
return {"available": True, "envs": envs, "task_default": settings.task_conda_env}
def disk_usage() -> dict:
usage = shutil.disk_usage(settings.source_root)
return {
"path": str(settings.source_root),
"total": usage.total,
"used": usage.used,
"free": usage.free,
}
def scan_results() -> list[dict]:
roots = [
settings.source_root / "DataSet_Public_outputs",
settings.source_root / "BestMode_Predict_Results_DataSet_Public",
settings.source_root / "Hardisk",
settings.source_root / "Seg_All_In_One_Analysis",
]
exts = {".csv", ".png", ".jpg", ".jpeg", ".svg", ".log", ".pth", ".pt"}
results: list[dict] = []
for root in roots:
if not root.exists():
continue
for path in root.rglob("*"):
if path.is_file() and path.suffix.lower() in exts:
try:
stat = path.stat()
results.append(
{
"name": path.name,
"path": str(path.resolve()),
"relative_path": str(path.resolve().relative_to(settings.source_root)),
"size": stat.st_size,
"modified": stat.st_mtime,
"kind": path.suffix.lower().lstrip("."),
}
)
except OSError:
continue
results.sort(key=lambda item: item["modified"], reverse=True)
return results[:1000]

View File

@@ -0,0 +1,14 @@
from __future__ import annotations
from ...commands import CommandSpec, bash
from ...config import settings
def build_system_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
if job_type == "system.backup":
return CommandSpec(bash(settings.source_root / "Back_Up.sh"), settings.source_root, "run legacy backup script")
if job_type == "mock.echo":
message = params.get("message", "Seg Data Server mock job")
return CommandSpec(["python", "-c", f"print({message!r})"], settings.project_root, "test job runner")
return None

View File

@@ -0,0 +1,2 @@
"""Weight sync and verification."""

View File

@@ -0,0 +1,153 @@
from __future__ import annotations
import hashlib
import json
import os
import shutil
import subprocess
from datetime import datetime, timezone
from pathlib import Path
from typing import Iterable
from ...config import settings
WEIGHT_EXTS = {".pt", ".pth", ".onnx", ".engine"}
def sha256_file(path: Path, chunk_size: int = 1024 * 1024 * 8) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
while True:
chunk = handle.read(chunk_size)
if not chunk:
break
digest.update(chunk)
return digest.hexdigest()
def iter_source_weights() -> Iterable[Path]:
project_root = settings.project_root.resolve()
for path in settings.source_root.rglob("*"):
if not path.is_file() or path.suffix.lower() not in WEIGHT_EXTS:
continue
try:
path.resolve().relative_to(project_root)
continue
except ValueError:
yield path
def classify_weight(path: Path) -> dict[str, str]:
try:
rel = str(path.resolve().relative_to(settings.source_root))
except ValueError:
rel = str(path)
lower = rel.lower()
if "yolo" in lower:
family = "yolo"
elif "mmseg" in lower or "my_local_model" in lower:
family = "mmseg"
elif "segmodel" in lower:
family = "segmodel"
else:
family = "misc"
if "best.pt" in lower or "best.pth" in lower:
role = "trained_best"
elif "last.pt" in lower or "last.pth" in lower:
role = "trained_last"
elif "pretrain" in lower or "my_local_model" in lower:
role = "pretrained"
else:
role = "weight"
return {"family": family, "role": role}
def copy_weight(src: Path, dst: Path, mode: str) -> None:
dst.parent.mkdir(parents=True, exist_ok=True)
if mode == "hardlink":
if dst.exists():
dst.unlink()
os.link(src, dst)
elif mode == "reflink":
subprocess.run(["cp", "--reflink=auto", "--preserve=timestamps", str(src), str(dst)], check=True)
else:
shutil.copy2(src, dst)
def sync_weights(mode: str = "copy", hash_files: bool = True, skip_existing: bool = True) -> dict:
files_dir = settings.weights_root / "files"
settings.weights_root.mkdir(parents=True, exist_ok=True)
entries = []
total_bytes = 0
for src in sorted(iter_source_weights()):
rel = src.resolve().relative_to(settings.source_root)
dst = files_dir / rel
stat = src.stat()
total_bytes += stat.st_size
copied = False
if not (skip_existing and dst.exists() and dst.stat().st_size == stat.st_size):
copy_weight(src, dst, mode)
copied = True
meta = classify_weight(src)
entry = {
"source_path": str(rel),
"stored_path": str(dst.resolve().relative_to(settings.project_root)),
"size": stat.st_size,
"family": meta["family"],
"role": meta["role"],
"copied": copied,
}
if hash_files:
entry["sha256"] = sha256_file(dst)
entries.append(entry)
manifest = {
"generated_at": datetime.now(timezone.utc).isoformat(),
"updated_at": datetime.now(timezone.utc).isoformat(),
"source_root": str(settings.source_root),
"mode": mode,
"count": len(entries),
"total_bytes": total_bytes,
"files": entries,
}
manifest_path = settings.weights_root / "manifest.json"
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
return manifest
def load_manifest() -> dict:
manifest_path = settings.weights_root / "manifest.json"
if not manifest_path.exists():
return {
"generated_at": None,
"source_root": str(settings.source_root),
"count": 0,
"total_bytes": 0,
"files": [],
}
return json.loads(manifest_path.read_text(encoding="utf-8"))
def verify_weights() -> dict:
manifest = load_manifest()
checked = []
ok_count = 0
for entry in manifest.get("files", []):
path = settings.project_root / entry["stored_path"]
exists = path.exists()
size_ok = exists and path.stat().st_size == entry.get("size")
hash_ok = None
if exists and "sha256" in entry:
hash_ok = sha256_file(path) == entry["sha256"]
ok = bool(exists and size_ok and (hash_ok is not False))
ok_count += int(ok)
checked.append(
{
"stored_path": entry["stored_path"],
"exists": exists,
"size_ok": size_ok,
"hash_ok": hash_ok,
"ok": ok,
}
)
return {"count": len(checked), "ok_count": ok_count, "items": checked}

View File

@@ -0,0 +1,2 @@
"""YOLO task wrappers."""

View File

@@ -0,0 +1,69 @@
from __future__ import annotations
from ...commands import CommandSpec, append_flag, bash, conda_python, required
from ...config import settings
YOLO_DIR = settings.source_root / "Seg_All_In_One_YoloModel"
VIDEO_YOLO_DIR = settings.source_root / "Seg_Predict_YoloModel"
def build_yolo_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
env = {"SEG_CONDA_ENV": conda_env}
if job_type == "yolo.train":
args = conda_python(conda_env, YOLO_DIR / "yolo_train.py")
append_flag(args, "--model", required(params, "model"))
return CommandSpec(args, YOLO_DIR, "train one Ultralytics YOLO segmentation model")
if job_type == "yolo.batch_train":
return CommandSpec(bash(YOLO_DIR / "yolo_train.sh"), YOLO_DIR, "run legacy YOLO batch training", env=env)
if job_type == "yolo.predict":
args = conda_python(conda_env, YOLO_DIR / "yolo_predict_V2.py")
append_flag(args, "--model", required(params, "model"))
append_flag(args, "--source", params.get("source"))
append_flag(args, "--pt_name", params.get("pt_name", "best.pt"))
append_flag(args, "--conf", params.get("conf", 0.2))
choice = str(params.get("run_choice", 1))
return CommandSpec(args, YOLO_DIR, "predict with one YOLO model", stdin_text=f"{choice}\n")
if job_type == "yolo.batch_predict":
args = bash(YOLO_DIR / "yolo_predict.sh")
append_flag(args, "--pt_name", params.get("pt_name", "best.pt"))
append_flag(args, "--conf", params.get("conf", 0.2))
append_flag(args, "--heatmap_method", params.get("heatmap_method"))
return CommandSpec(args, YOLO_DIR, "run legacy YOLO batch prediction", env=env)
if job_type == "yolo.heatmap":
args = conda_python(conda_env, YOLO_DIR / "yolo_predict_visualize_nn.py")
append_flag(args, "--model", required(params, "model"))
append_flag(args, "--target_layers", params.get("target_layers", "default"))
append_flag(args, "--cam_method", params.get("cam_method", "All"))
append_flag(args, "--pt_name", params.get("pt_name", "best.pt"))
choice = str(params.get("run_choice", 1))
return CommandSpec(args, YOLO_DIR, "generate YOLO heatmaps", stdin_text=f"{choice}\n")
if job_type == "yolo.compare":
args = conda_python(conda_env, YOLO_DIR / "yolo_predict_V2_compare_all.py")
append_flag(args, "--pt_name", params.get("pt_name", "all"))
return CommandSpec(args, YOLO_DIR, "compare all YOLO prediction outputs")
if job_type == "yolo.raw_mask_check":
args = conda_python(conda_env, YOLO_DIR / "yolo_predict_raw_masks_check.py")
append_flag(args, "--pt_name", params.get("pt_name", "best.pt"))
return CommandSpec(args, YOLO_DIR, "check YOLO raw mask completeness")
if job_type == "yolo.copy_best":
args = bash(YOLO_DIR / "Tool_Yolo_Copy_Best_Model.sh")
append_flag(args, "--pt_name", params.get("pt_name", "best.pt"))
return CommandSpec(args, YOLO_DIR, "copy YOLO best weights into prediction area")
if job_type == "yolo.video_visible":
return CommandSpec(conda_python(conda_env, VIDEO_YOLO_DIR / "yolo_Seg_Video-V1-Visible.py"), VIDEO_YOLO_DIR, "render visible YOLO video prediction")
if job_type == "yolo.video_unvisible":
return CommandSpec(conda_python(conda_env, VIDEO_YOLO_DIR / "yolo_Seg_Video-V2-UnVisible.py"), VIDEO_YOLO_DIR, "render invisible/headless YOLO video prediction")
return None