Expand Seg task coverage and coverage dashboard

This commit is contained in:
2026-06-30 12:54:25 +08:00
parent dd7b7384ec
commit 7a43303f15
18 changed files with 849 additions and 28 deletions

View File

@@ -6,6 +6,7 @@ from .mmseg.tasks import build_mmseg_task
from .segmodel.tasks import build_segmodel_task
from .system.tasks import build_system_task
from .yolo.tasks import build_yolo_task
from .visual.tasks import build_visual_task
def build_module_task(job_type: str, params: dict, conda_env: str):
@@ -13,6 +14,7 @@ def build_module_task(job_type: str, params: dict, conda_env: str):
build_dataset_task,
build_segmodel_task,
build_yolo_task,
build_visual_task,
build_mmseg_task,
build_analysis_task,
build_system_task,
@@ -21,4 +23,3 @@ def build_module_task(job_type: str, params: dict, conda_env: str):
if spec is not None:
return spec
return None

View File

@@ -16,6 +16,24 @@ def _dataset_script(name: str) -> Path:
return DATASET_TOOL_DIR / name
def _append_label_rebuild_flags(args: list[str], params: dict) -> None:
append_flag(args, "-src_fold", params.get("src_label_fold"))
append_flag(args, "-save_pro_fold", params.get("save_pro_label_fold"))
append_flag(args, "-save_GT_fold", params.get("save_GT_label_fold"))
append_flag(args, "-fold_search_depth", params.get("Label_Max_Search_layer"))
append_flag(args, "-pro_suffix_name", params.get("pro_append_name"))
append_flag(args, "-GT_suffix_name", params.get("GT_append_name"))
append_flag(args, "-GT_channel", params.get("GT_channel"))
append_flag(args, "-back_gnd_color", params.get("back_gnd_color"))
append_flag(args, "-first_class_color", params.get("first_class_color"))
append_flag(args, "-pic_type", params.get("pic_type"))
append_flag(args, "-Max_width", params.get("Max_width"))
append_flag(args, "-Rebuild_from", params.get("Rebuild_from"))
append_flag(args, "-Rebuild_to", params.get("Rebuild_to"))
if "save_process_pics" in params:
append_flag(args, "-save_process_pics", str(params["save_process_pics"]))
def build_dataset_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
if job_type == "dataset.rename":
args = bash(_dataset_script("1_rename_pics.sh"))
@@ -38,6 +56,18 @@ def build_dataset_task(job_type: str, params: dict, conda_env: str) -> CommandSp
append_flag(args, "-h", params.get("height", 1080))
return CommandSpec(args, DATASET_TOOL_DIR, "resize and reformat image/label folders")
if job_type == "dataset.resize_single":
script = _dataset_script("2_2_Resize.py")
args = conda_python(
conda_env,
script,
required(params, "folder"),
str(bool(params.get("nearest", False))),
params.get("width", 1920),
params.get("height", 1080),
)
return CommandSpec(args, DATASET_TOOL_DIR, "resize one image folder with the legacy Python tool")
if job_type == "dataset.pair":
args = bash(_dataset_script("3_pair_ori_label.sh"))
append_flag(args, "-i", required(params, "image_dir"))
@@ -51,6 +81,12 @@ def build_dataset_task(job_type: str, params: dict, conda_env: str) -> CommandSp
append_flag(args, "-l", required(params, "label_dir"))
return CommandSpec(args, DATASET_TOOL_DIR, "rebuild color labels into GT masks")
if job_type in {"dataset.deal_labels", "dataset.deal_labels_old"}:
script_name = "4_deal_labels_old(老版程序).py" if job_type.endswith("_old") else "4_deal_labels.py"
args = conda_python(conda_env, _dataset_script(script_name))
_append_label_rebuild_flags(args, params)
return CommandSpec(args, DATASET_TOOL_DIR, "run legacy label rebuild/conversion script")
if job_type == "dataset.stack":
args = bash(_dataset_script("5_TOOL_stack_pics.sh"))
append_flag(args, "-i", required(params, "image_dir"))
@@ -61,6 +97,17 @@ def build_dataset_task(job_type: str, params: dict, conda_env: str) -> CommandSp
append_flag(args, "-s", params.get("suffix", ""))
return CommandSpec(args, DATASET_TOOL_DIR, "overlay image and label for inspection")
if job_type == "dataset.stack_single":
args = conda_python(
conda_env,
_dataset_script("5_stack_picture.py"),
required(params, "image_path"),
required(params, "label_path"),
required(params, "result_dir"),
params.get("alpha", 0.3),
)
return CommandSpec(args, DATASET_TOOL_DIR, "overlay one image and label pair")
if job_type == "dataset.stitch":
args = bash(_dataset_script("6_TOOL_stitch_pics.sh"))
append_flag(args, "-i", required(params, "image_dir"))
@@ -68,6 +115,52 @@ def build_dataset_task(job_type: str, params: dict, conda_env: str) -> CommandSp
append_flag(args, "-r", required(params, "result_dir"))
return CommandSpec(args, DATASET_TOOL_DIR, "stitch image and label panels")
if job_type == "dataset.stitch_single":
args = conda_python(
conda_env,
_dataset_script("6_stitch_picture.py"),
required(params, "image_path"),
required(params, "label_path"),
required(params, "result_dir"),
params.get("relative_pos", "up_down"),
)
return CommandSpec(args, DATASET_TOOL_DIR, "stitch one image and label pair")
if job_type == "dataset.run_wizard":
args = bash(_dataset_script("Seg_data_run.sh"))
append_flag(args, "-i", required(params, "image_dir"))
append_flag(args, "-l", required(params, "label_dir"))
return CommandSpec(args, DATASET_TOOL_DIR, "run legacy interactive dataset wizard", stdin_text=params.get("stdin_text"))
if job_type == "dataset.stack_pair_check":
args = conda_python(conda_env, STACK_TOOL_DIR / "1_check_picture_pair.py")
append_flag(args, "-i", required(params, "image_dir"))
append_flag(args, "-l", required(params, "label_dir"))
append_flag(args, "-p", params.get("prefix", ""))
append_flag(args, "-s", params.get("suffix", ""))
return CommandSpec(args, STACK_TOOL_DIR, "check image/label pair names with stack tool")
if job_type == "dataset.stack_tool_batch":
args = bash(STACK_TOOL_DIR / "2_TOOL_stack_pics.sh")
append_flag(args, "-i", required(params, "image_dir"))
append_flag(args, "-l", required(params, "label_dir"))
append_flag(args, "-r", required(params, "result_dir"))
append_flag(args, "-a", params.get("alpha", 0.3))
append_flag(args, "-p", params.get("prefix", ""))
append_flag(args, "-s", params.get("suffix", ""))
return CommandSpec(args, STACK_TOOL_DIR, "run standalone stack tool batch overlay")
if job_type == "dataset.stack_tool_single":
args = conda_python(
conda_env,
STACK_TOOL_DIR / "2_stack_picture.py",
required(params, "image_path"),
required(params, "label_path"),
required(params, "result_dir"),
params.get("alpha", 0.3),
)
return CommandSpec(args, STACK_TOOL_DIR, "run standalone stack tool for one pair")
if job_type == "dataset.video_frames":
script = VIDEO_DIR / "1_Save_Frame_V2.py"
args = conda_python(conda_env, script)
@@ -82,7 +175,55 @@ def build_dataset_task(job_type: str, params: dict, conda_env: str) -> CommandSp
args = conda_python(conda_env, script)
append_flag(args, "-i", required(params, "image_dir"))
append_flag(args, "-l", required(params, "label_dir"))
append_flag(args, "-p", params.get("prefix", ""))
append_flag(args, "-s", params.get("suffix", ""))
if params.get("yes"):
args.append("-y")
return CommandSpec(args, YOLO_DATASET_DIR, "check YOLO image/label pairs")
return None
if job_type == "dataset.yolo_stack":
args = bash(YOLO_DATASET_DIR / "0_2_TOOL_stack_pics.sh")
append_flag(args, "-i", required(params, "image_dir"))
append_flag(args, "-l", required(params, "label_dir"))
append_flag(args, "-r", required(params, "result_dir"))
append_flag(args, "-a", params.get("alpha", 0.3))
append_flag(args, "-p", params.get("prefix", ""))
append_flag(args, "-s", params.get("suffix", ""))
return CommandSpec(args, YOLO_DATASET_DIR, "overlay YOLO dataset image/label pairs")
if job_type == "dataset.yolo_stack_single":
args = conda_python(
conda_env,
YOLO_DATASET_DIR / "0_2_stack_picture.py",
required(params, "image_path"),
required(params, "label_path"),
required(params, "result_dir"),
params.get("alpha", 0.3),
)
return CommandSpec(args, YOLO_DATASET_DIR, "overlay one YOLO dataset image/label pair")
if job_type == "dataset.yolo_rebuild_labels":
args = conda_python(conda_env, YOLO_DATASET_DIR / "1_deal_labels.py")
_append_label_rebuild_flags(args, params)
return CommandSpec(args, YOLO_DATASET_DIR, "rebuild labels for YOLO dataset generation")
if job_type == "dataset.yolo_txt_ori":
return CommandSpec(conda_python(conda_env, YOLO_DATASET_DIR / "2_Check_and_Gen_Txt_Label_ori_label.py"), YOLO_DATASET_DIR, "generate YOLO txt labels preserving original class ids")
if job_type == "dataset.yolo_txt_sort":
return CommandSpec(conda_python(conda_env, YOLO_DATASET_DIR / "2_Check_and_Gen_Txt_Label_sort_label.py"), YOLO_DATASET_DIR, "generate YOLO txt labels with sorted class ids")
if job_type == "dataset.yolo_convert_png":
args = conda_python(conda_env, YOLO_DATASET_DIR / "Tool_convert_bmp_jpg_to_png.py", required(params, "folder"))
if params.get("delete_source"):
args.append("--delete-source")
append_flag(args, "--workers", params.get("workers"))
return CommandSpec(args, YOLO_DATASET_DIR, "convert YOLO dataset images to PNG")
if job_type == "dataset.yolo_resize":
args = conda_python(conda_env, YOLO_DATASET_DIR / "Tool_resize_pics.py", required(params, "folder"))
append_flag(args, "--size", params.get("size"))
append_flag(args, "--workers", params.get("workers"))
return CommandSpec(args, YOLO_DATASET_DIR, "resize YOLO dataset images recursively")
return None

View File

@@ -1,6 +1,6 @@
from __future__ import annotations
from ...commands import CommandSpec, append_flag, conda_python, required
from ...commands import CommandSpec, append_flag, bash, conda_python, required
from ...config import settings
@@ -49,6 +49,12 @@ def build_mmseg_task(job_type: str, params: dict, conda_env: str) -> CommandSpec
if job_type == "mmseg.generate_data":
return CommandSpec(conda_python(conda_env, MY_DIR / "1_Initial_Data_All_data_from_1_Data_Parameter-V2.py"), MMSEG_DIR, "generate MMSeg dataset configs from JSON parameters")
if job_type == "mmseg.generate_data_v1":
return CommandSpec(conda_python(conda_env, MY_DIR / "1_Initial_Data_All_data_from_1_Data_Parameter-V1.py"), MMSEG_DIR, "generate MMSeg dataset configs with V1 flow")
if job_type == "mmseg.generate_data_legacy":
return CommandSpec(conda_python(conda_env, MY_DIR / "1_Initial_Data_All-ori.py"), MMSEG_DIR, "run legacy original MMSeg dataset config generator")
if job_type == "mmseg.generate_alg":
script = MY_DIR / "2_Initial_Alg_All_data_from_2_Alg_Program-V2.py"
return CommandSpec(
@@ -58,6 +64,18 @@ def build_mmseg_task(job_type: str, params: dict, conda_env: str) -> CommandSpec
stdin_text=_stdin_for_generate_alg(params),
)
if job_type == "mmseg.generate_alg_v1":
script = MY_DIR / "2_Initial_Alg_All_data_from_2_Alg_Program-V1.py"
return CommandSpec(
conda_python(conda_env, script),
MMSEG_DIR,
"generate MMSeg algorithm config and training command with V1 flow",
stdin_text=_stdin_for_generate_alg(params),
)
if job_type == "mmseg.generate_alg_legacy":
return CommandSpec(conda_python(conda_env, MY_DIR / "2_Initial_Alg_All-ori-old.py"), MMSEG_DIR, "run legacy original MMSeg algorithm config generator")
if job_type == "mmseg.train":
config_path = required(params, "config")
args = conda_python(conda_env, MMSEG_DIR / "tools" / "train.py", config_path)
@@ -71,6 +89,11 @@ def build_mmseg_task(job_type: str, params: dict, conda_env: str) -> CommandSpec
stdin = f"{params.get('dataset_choice', 1)}\n{params.get('algorithm_choice', 0)}\n"
return CommandSpec(args, MMSEG_DIR, "extract best MMSeg metrics from logs", stdin_text=stdin)
if job_type == "mmseg.metrics_v1":
args = conda_python(conda_env, MY_DIR / "4_2_predict_matrics_from_log_V1.py")
stdin = f"{params.get('dataset_choice', 1)}\n{params.get('algorithm_choice', 0)}\n"
return CommandSpec(args, MMSEG_DIR, "extract best MMSeg metrics from logs with V1 script", stdin_text=stdin)
if job_type == "mmseg.flops_fps":
args = conda_python(conda_env, MY_DIR / "4_1_predict_params_FLOPs_FPS_V2.py")
append_flag(args, "--input_dir", params.get("input_dir", "../Hardisk"))
@@ -81,6 +104,11 @@ def build_mmseg_task(job_type: str, params: dict, conda_env: str) -> CommandSpec
stdin += f"{params['shape_h']}\n{params['shape_w']}\n"
return CommandSpec(args, MMSEG_DIR, "calculate MMSeg FLOPs/Params/FPS", stdin_text=stdin)
if job_type == "mmseg.flops_fps_v1":
args = conda_python(conda_env, MY_DIR / "4_1_predict_params_and_FLOPs_V1.py")
stdin = f"{params.get('dataset_choice', 1)}\n{params.get('algorithm_choice', 0)}\n"
return CommandSpec(args, MMSEG_DIR, "calculate MMSeg FLOPs/Params/FPS with V1 script", stdin_text=stdin)
if job_type == "mmseg.draw":
return CommandSpec(conda_python(conda_env, MY_DIR / "4_3_predict_draw_pictures_and_tabels.py"), MMSEG_DIR, "generate MMSeg prediction pictures and tables")
@@ -90,5 +118,13 @@ def build_mmseg_task(job_type: str, params: dict, conda_env: str) -> CommandSpec
if job_type == "mmseg.delete_epoch":
return CommandSpec(conda_python(conda_env, MY_DIR / "3_Find_And_Delete_Special_Epoch.py"), MMSEG_DIR, "find and delete selected epoch checkpoints")
return None
if job_type == "mmseg.copy_result":
return CommandSpec(bash(MY_DIR / "3_Tool_Copy_Result_To_Hardisk.sh"), MMSEG_DIR, "copy MMSeg results to Hardisk")
if job_type == "mmseg.predict_v1":
return CommandSpec(conda_python(conda_env, MY_DIR / "x4_Predict_V1-.py"), MMSEG_DIR, "run MMSeg prediction V1 helper")
if job_type == "mmseg.predict_v2":
return CommandSpec(conda_python(conda_env, MY_DIR / "x4_Predict_V2-.py"), MMSEG_DIR, "run MMSeg prediction V2 helper")
return None

View File

@@ -31,11 +31,39 @@ def build_segmodel_task(job_type: str, params: dict, conda_env: str) -> CommandS
script = SEGMODEL_DIR / params.get("script", "2_predict_params_and_FLOPs_V2.py")
return CommandSpec(conda_python(conda_env, script), SEGMODEL_DIR, "calculate SegModel params/FLOPs/FPS")
if job_type == "segmodel.params_flops":
args = conda_python(conda_env, SEGMODEL_DIR / "Tool_get_params_and_FLOPs.py")
append_flag(args, "-a", required(params, "architecture"))
shape = params.get("shape", [512, 512])
if isinstance(shape, str):
shape = [part for part in shape.replace(",", " ").split() if part]
if shape:
args.append("--shape")
args.extend(str(item) for item in shape)
return CommandSpec(args, SEGMODEL_DIR, "calculate SegModel params and FLOPs for one architecture")
if job_type == "segmodel.benchmark":
args = conda_python(conda_env, SEGMODEL_DIR / "Tool_benchmark_smp.py")
append_flag(args, "-a", required(params, "architecture"))
append_flag(args, "-c", params.get("checkpoint"))
shape = params.get("shape", [512, 512])
if isinstance(shape, str):
shape = [part for part in shape.replace(",", " ").split() if part]
if shape:
args.append("--shape")
args.extend(str(item) for item in shape)
append_flag(args, "--repeat-times", params.get("repeat_times"))
append_flag(args, "--log-interval", params.get("log_interval"))
stdin = None if params.get("checkpoint") else params.get("stdin_text")
return CommandSpec(args, SEGMODEL_DIR, "benchmark SegModel FPS/latency", stdin_text=stdin)
if job_type == "segmodel.raw_mask_check":
return CommandSpec(conda_python(conda_env, SEGMODEL_DIR / "1_predict_raw_masks_check.py"), SEGMODEL_DIR, "check SegModel raw mask completeness")
if job_type == "segmodel.metrics":
return CommandSpec(conda_python(conda_env, SEGMODEL_DIR / "3_predict_matrics_from_log.py"), SEGMODEL_DIR, "parse SegModel training/prediction metrics")
return None
if job_type == "segmodel.copy_best":
return CommandSpec(bash(SEGMODEL_DIR / "Tool_Copy_Best_Model.sh"), SEGMODEL_DIR, "copy best SegModel weights to prediction area")
return None

View File

@@ -7,8 +7,9 @@ from ...config import settings
def build_system_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
if job_type == "system.backup":
return CommandSpec(bash(settings.source_root / "Back_Up.sh"), settings.source_root, "run legacy backup script")
if job_type == "system.check_graph_card":
return CommandSpec(bash(settings.source_root / "Check_Graph_Card.sh"), settings.source_root, "run legacy GPU card detection script")
if job_type == "mock.echo":
message = params.get("message", "Seg Data Server mock job")
return CommandSpec(["python", "-c", f"print({message!r})"], settings.project_root, "test job runner")
return None

View File

@@ -0,0 +1,2 @@
"""Visual tooling task wrappers."""

View File

@@ -0,0 +1,59 @@
from __future__ import annotations
from ...commands import CommandSpec, append_flag, conda_python
from ...config import settings
from ..dataset.tasks import _append_label_rebuild_flags
VISUAL_DIR = settings.source_root / "Tool-可视化"
VISUAL_LABEL_DIR = VISUAL_DIR / "0_图片Labels生成"
def build_visual_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
if job_type == "visual.train":
return CommandSpec(conda_python(conda_env, VISUAL_DIR / "train.py"), VISUAL_DIR, "run standalone YOLO visual training script")
if job_type == "visual.inference":
return CommandSpec(conda_python(conda_env, VISUAL_DIR / "inference.py"), VISUAL_DIR, "run standalone YOLO visual inference script")
if job_type == "visual.fps":
args = conda_python(conda_env, VISUAL_DIR / "get_FPS.py")
append_flag(args, "--weights", params.get("weights", "yolov8n.pt"))
append_flag(args, "--batch", params.get("batch", 1))
imgs = params.get("imgs", [640, 640])
if isinstance(imgs, str):
imgs = [part for part in imgs.replace(",", " ").split() if part]
if imgs:
args.append("--imgs")
args.extend(str(item) for item in imgs)
append_flag(args, "--device", params.get("device", ""))
append_flag(args, "--warmup", params.get("warmup"))
append_flag(args, "--testtime", params.get("testtime"))
if params.get("half"):
args.append("--half")
return CommandSpec(args, VISUAL_DIR, "measure YOLO visual model FPS")
if job_type == "visual.yolo11_heatmap_v1":
return CommandSpec(conda_python(conda_env, VISUAL_DIR / "yolov11_heatmap_V1.py"), VISUAL_DIR, "run standalone YOLOv11 heatmap V1")
if job_type == "visual.yolo11_heatmap_v2":
return CommandSpec(conda_python(conda_env, VISUAL_DIR / "yolov11_heatmap_V2.py"), VISUAL_DIR, "run standalone YOLOv11 heatmap V2")
if job_type == "visual.label_ori":
return CommandSpec(conda_python(conda_env, VISUAL_DIR / "Tool_Check_and_Gen_Txt_Label_ori_label.py"), VISUAL_DIR, "generate YOLO txt labels from original label ids")
if job_type == "visual.label_sort":
return CommandSpec(conda_python(conda_env, VISUAL_DIR / "Tool_Check_and_Gen_Txt_Label_sort_label.py"), VISUAL_DIR, "generate YOLO txt labels from sorted label ids")
if job_type == "visual.gen_8bit_png":
return CommandSpec(conda_python(conda_env, VISUAL_DIR / "Tool_Gen_8_Bit_PNG[没用,不认].py"), VISUAL_DIR, "generate 8-bit PNG labels with legacy visual tool")
if job_type == "visual.deal_labels":
args = conda_python(conda_env, VISUAL_LABEL_DIR / "4_deal_labels.py")
_append_label_rebuild_flags(args, params)
return CommandSpec(args, VISUAL_LABEL_DIR, "run visual label rebuild/conversion script")
if job_type == "visual.tool_deal_labels_demo":
return CommandSpec(conda_python(conda_env, VISUAL_LABEL_DIR / "Tool_deal_labels.py"), VISUAL_LABEL_DIR, "run visual label helper demo script")
return None

View File

@@ -28,6 +28,15 @@ def build_yolo_task(job_type: str, params: dict, conda_env: str) -> CommandSpec
choice = str(params.get("run_choice", 1))
return CommandSpec(args, YOLO_DIR, "predict with one YOLO model", stdin_text=f"{choice}\n")
if job_type == "yolo.predict_v1":
args = conda_python(conda_env, YOLO_DIR / "yolo_predict_V1.py")
append_flag(args, "--model", required(params, "model"))
append_flag(args, "--source", params.get("source"))
append_flag(args, "--pt_name", params.get("pt_name", "best.pt"))
append_flag(args, "--conf", params.get("conf", 0.2))
choice = str(params.get("run_choice", 1))
return CommandSpec(args, YOLO_DIR, "predict with legacy YOLO V1 script", stdin_text=f"{choice}\n")
if job_type == "yolo.batch_predict":
args = bash(YOLO_DIR / "yolo_predict.sh")
append_flag(args, "--pt_name", params.get("pt_name", "best.pt"))
@@ -65,5 +74,7 @@ def build_yolo_task(job_type: str, params: dict, conda_env: str) -> CommandSpec
if job_type == "yolo.video_unvisible":
return CommandSpec(conda_python(conda_env, VIDEO_YOLO_DIR / "yolo_Seg_Video-V2-UnVisible.py"), VIDEO_YOLO_DIR, "render invisible/headless YOLO video prediction")
return None
if job_type == "yolo.layer_tester":
return CommandSpec(conda_python(conda_env, YOLO_DIR / "Yolo可视化测试" / "yolo_layer_tester.py"), YOLO_DIR, "test YOLO heatmap target layers")
return None