Initial Seg Data Server Net platform
This commit is contained in:
24
backend/app/modules/__init__.py
Normal file
24
backend/app/modules/__init__.py
Normal file
@@ -0,0 +1,24 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .analysis.tasks import build_analysis_task
|
||||
from .dataset.tasks import build_dataset_task
|
||||
from .mmseg.tasks import build_mmseg_task
|
||||
from .segmodel.tasks import build_segmodel_task
|
||||
from .system.tasks import build_system_task
|
||||
from .yolo.tasks import build_yolo_task
|
||||
|
||||
|
||||
def build_module_task(job_type: str, params: dict, conda_env: str):
|
||||
for builder in (
|
||||
build_dataset_task,
|
||||
build_segmodel_task,
|
||||
build_yolo_task,
|
||||
build_mmseg_task,
|
||||
build_analysis_task,
|
||||
build_system_task,
|
||||
):
|
||||
spec = builder(job_type, params, conda_env)
|
||||
if spec is not None:
|
||||
return spec
|
||||
return None
|
||||
|
||||
2
backend/app/modules/analysis/__init__.py
Normal file
2
backend/app/modules/analysis/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
"""Analysis task wrappers."""
|
||||
|
||||
18
backend/app/modules/analysis/tasks.py
Normal file
18
backend/app/modules/analysis/tasks.py
Normal file
@@ -0,0 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from ...commands import CommandSpec, append_flag, conda_python
|
||||
from ...config import settings
|
||||
|
||||
|
||||
ANALYSIS_DIR = settings.source_root / "Seg_All_In_One_Analysis"
|
||||
|
||||
|
||||
def build_analysis_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
|
||||
if job_type != "analysis.all":
|
||||
return None
|
||||
args = conda_python(conda_env, ANALYSIS_DIR / "1_Analysis_All.py")
|
||||
append_flag(args, "--input_dir", params.get("input_dir", "../BestMode_Predict_Results_DataSet_Public"))
|
||||
append_flag(args, "--output_dir", params.get("output_dir", "./"))
|
||||
stdin = f"{params.get('dataset_choice', 1)}\n"
|
||||
return CommandSpec(args, ANALYSIS_DIR, "merge SegModel/MMSeg metrics and generate plots", stdin_text=stdin)
|
||||
|
||||
2
backend/app/modules/dataset/__init__.py
Normal file
2
backend/app/modules/dataset/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
"""Dataset task wrappers."""
|
||||
|
||||
88
backend/app/modules/dataset/tasks.py
Normal file
88
backend/app/modules/dataset/tasks.py
Normal file
@@ -0,0 +1,88 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from ...commands import CommandSpec, append_flag, bash, conda_python, required
|
||||
from ...config import settings
|
||||
|
||||
|
||||
DATASET_TOOL_DIR = settings.source_root / "DataSet_Own" / "1. 图片预处理(内含使用手册)"
|
||||
STACK_TOOL_DIR = settings.source_root / "Tool-图片堆叠"
|
||||
VIDEO_DIR = settings.source_root / "Seg_Predict_Own_Video_V2"
|
||||
YOLO_DATASET_DIR = settings.source_root / "Seg_All_In_One_YoloModel" / "Yolo数据集构建"
|
||||
|
||||
|
||||
def _dataset_script(name: str) -> Path:
|
||||
return DATASET_TOOL_DIR / name
|
||||
|
||||
|
||||
def build_dataset_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
|
||||
if job_type == "dataset.rename":
|
||||
args = bash(_dataset_script("1_rename_pics.sh"))
|
||||
append_flag(args, "-i", required(params, "image_dir"))
|
||||
append_flag(args, "-l", required(params, "label_dir"))
|
||||
return CommandSpec(args, DATASET_TOOL_DIR, "rename and normalize image/label names")
|
||||
|
||||
if job_type == "dataset.to_png":
|
||||
script = _dataset_script("2_1_Trans_to_png.py")
|
||||
args = conda_python(conda_env, script)
|
||||
append_flag(args, "-i", params.get("input_dir"))
|
||||
append_flag(args, "-o", params.get("output_dir"))
|
||||
return CommandSpec(args, DATASET_TOOL_DIR, "convert images to png")
|
||||
|
||||
if job_type == "dataset.resize":
|
||||
args = bash(_dataset_script("2_reformate_pics.sh"))
|
||||
append_flag(args, "-i", params.get("image_dir"))
|
||||
append_flag(args, "-l", params.get("label_dir"))
|
||||
append_flag(args, "-w", params.get("width", 1920))
|
||||
append_flag(args, "-h", params.get("height", 1080))
|
||||
return CommandSpec(args, DATASET_TOOL_DIR, "resize and reformat image/label folders")
|
||||
|
||||
if job_type == "dataset.pair":
|
||||
args = bash(_dataset_script("3_pair_ori_label.sh"))
|
||||
append_flag(args, "-i", required(params, "image_dir"))
|
||||
append_flag(args, "-l", required(params, "label_dir"))
|
||||
append_flag(args, "-p", params.get("prefix", ""))
|
||||
append_flag(args, "-s", params.get("suffix", ""))
|
||||
return CommandSpec(args, DATASET_TOOL_DIR, "check image and label pairing")
|
||||
|
||||
if job_type == "dataset.rebuild_labels":
|
||||
args = bash(_dataset_script("4_rebuild_labels.sh"))
|
||||
append_flag(args, "-l", required(params, "label_dir"))
|
||||
return CommandSpec(args, DATASET_TOOL_DIR, "rebuild color labels into GT masks")
|
||||
|
||||
if job_type == "dataset.stack":
|
||||
args = bash(_dataset_script("5_TOOL_stack_pics.sh"))
|
||||
append_flag(args, "-i", required(params, "image_dir"))
|
||||
append_flag(args, "-l", required(params, "label_dir"))
|
||||
append_flag(args, "-r", required(params, "result_dir"))
|
||||
append_flag(args, "-a", params.get("alpha", 0.3))
|
||||
append_flag(args, "-p", params.get("prefix", ""))
|
||||
append_flag(args, "-s", params.get("suffix", ""))
|
||||
return CommandSpec(args, DATASET_TOOL_DIR, "overlay image and label for inspection")
|
||||
|
||||
if job_type == "dataset.stitch":
|
||||
args = bash(_dataset_script("6_TOOL_stitch_pics.sh"))
|
||||
append_flag(args, "-i", required(params, "image_dir"))
|
||||
append_flag(args, "-l", required(params, "label_dir"))
|
||||
append_flag(args, "-r", required(params, "result_dir"))
|
||||
return CommandSpec(args, DATASET_TOOL_DIR, "stitch image and label panels")
|
||||
|
||||
if job_type == "dataset.video_frames":
|
||||
script = VIDEO_DIR / "1_Save_Frame_V2.py"
|
||||
args = conda_python(conda_env, script)
|
||||
append_flag(args, "--video", required(params, "video"))
|
||||
append_flag(args, "--interval", params.get("interval", 0.5))
|
||||
append_flag(args, "--resize", params.get("resize"))
|
||||
append_flag(args, "--output_dir", params.get("output_dir"))
|
||||
return CommandSpec(args, VIDEO_DIR, "extract video frames into DataSet_Public layout")
|
||||
|
||||
if job_type == "dataset.yolo_check_pairs":
|
||||
script = YOLO_DATASET_DIR / "0_1_check_picture_pair.py"
|
||||
args = conda_python(conda_env, script)
|
||||
append_flag(args, "-i", required(params, "image_dir"))
|
||||
append_flag(args, "-l", required(params, "label_dir"))
|
||||
return CommandSpec(args, YOLO_DATASET_DIR, "check YOLO image/label pairs")
|
||||
|
||||
return None
|
||||
|
||||
2
backend/app/modules/mmseg/__init__.py
Normal file
2
backend/app/modules/mmseg/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
"""MMSeg task wrappers."""
|
||||
|
||||
94
backend/app/modules/mmseg/tasks.py
Normal file
94
backend/app/modules/mmseg/tasks.py
Normal file
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from ...commands import CommandSpec, append_flag, conda_python, required
|
||||
from ...config import settings
|
||||
|
||||
|
||||
MMSEG_DIR = settings.source_root / "Seg_All_In_One_MMSeg"
|
||||
MY_DIR = MMSEG_DIR / "My_All_In_One"
|
||||
|
||||
|
||||
def _stdin_for_generate_alg(params: dict) -> str:
|
||||
lines = [
|
||||
str(params.get("dataset_choice", 1)),
|
||||
str(params.get("gpu_count", 1)),
|
||||
]
|
||||
gpu_ids = params.get("gpu_ids", [0])
|
||||
if isinstance(gpu_ids, str):
|
||||
gpu_ids = [part.strip() for part in gpu_ids.split(",") if part.strip()]
|
||||
for index in range(int(params.get("gpu_count", len(gpu_ids) or 1))):
|
||||
lines.append(str(gpu_ids[index] if index < len(gpu_ids) else 0))
|
||||
|
||||
mode = str(params.get("schedule_mode", 2))
|
||||
lines.append(mode)
|
||||
if mode == "1":
|
||||
lines.extend(
|
||||
[
|
||||
str(params.get("train_k", 40)),
|
||||
str(params.get("check_count", 10)),
|
||||
str(params.get("logger_interval", 50)),
|
||||
]
|
||||
)
|
||||
else:
|
||||
lines.extend(
|
||||
[
|
||||
str(params.get("max_epochs", 300)),
|
||||
str(params.get("val_interval", 1)),
|
||||
str(params.get("checkpoint_interval", 10)),
|
||||
str(params.get("logger_interval", "")),
|
||||
]
|
||||
)
|
||||
lines.append(str(params.get("algorithm_choice", 1)))
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def build_mmseg_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
|
||||
if job_type == "mmseg.init_weights":
|
||||
return CommandSpec(conda_python(conda_env, MY_DIR / "0_Initial_Save_All_Model_locally.py"), MMSEG_DIR, "download/save MMSeg pretrained weights locally")
|
||||
|
||||
if job_type == "mmseg.generate_data":
|
||||
return CommandSpec(conda_python(conda_env, MY_DIR / "1_Initial_Data_All_data_from_1_Data_Parameter-V2.py"), MMSEG_DIR, "generate MMSeg dataset configs from JSON parameters")
|
||||
|
||||
if job_type == "mmseg.generate_alg":
|
||||
script = MY_DIR / "2_Initial_Alg_All_data_from_2_Alg_Program-V2.py"
|
||||
return CommandSpec(
|
||||
conda_python(conda_env, script),
|
||||
MMSEG_DIR,
|
||||
"generate MMSeg algorithm config and training command",
|
||||
stdin_text=_stdin_for_generate_alg(params),
|
||||
)
|
||||
|
||||
if job_type == "mmseg.train":
|
||||
config_path = required(params, "config")
|
||||
args = conda_python(conda_env, MMSEG_DIR / "tools" / "train.py", config_path)
|
||||
append_flag(args, "--work-dir", params.get("work_dir"))
|
||||
return CommandSpec(args, MMSEG_DIR, "train MMSeg model")
|
||||
|
||||
if job_type == "mmseg.metrics":
|
||||
args = conda_python(conda_env, MY_DIR / "4_2_predict_matrics_from_log_V2.py")
|
||||
append_flag(args, "--input_dir", params.get("input_dir", "../Hardisk"))
|
||||
append_flag(args, "--output_dir", params.get("output_dir", "../BestMode_Predict_Results_DataSet_Public"))
|
||||
stdin = f"{params.get('dataset_choice', 1)}\n{params.get('algorithm_choice', 0)}\n"
|
||||
return CommandSpec(args, MMSEG_DIR, "extract best MMSeg metrics from logs", stdin_text=stdin)
|
||||
|
||||
if job_type == "mmseg.flops_fps":
|
||||
args = conda_python(conda_env, MY_DIR / "4_1_predict_params_FLOPs_FPS_V2.py")
|
||||
append_flag(args, "--input_dir", params.get("input_dir", "../Hardisk"))
|
||||
append_flag(args, "--output_dir", params.get("output_dir", "../BestMode_Predict_Results_DataSet_Public"))
|
||||
append_flag(args, "--repeat-times", params.get("repeat_times", 3))
|
||||
stdin = f"{params.get('dataset_choice', 1)}\n{params.get('algorithm_choice', 0)}\n"
|
||||
if "shape_h" in params and "shape_w" in params:
|
||||
stdin += f"{params['shape_h']}\n{params['shape_w']}\n"
|
||||
return CommandSpec(args, MMSEG_DIR, "calculate MMSeg FLOPs/Params/FPS", stdin_text=stdin)
|
||||
|
||||
if job_type == "mmseg.draw":
|
||||
return CommandSpec(conda_python(conda_env, MY_DIR / "4_3_predict_draw_pictures_and_tabels.py"), MMSEG_DIR, "generate MMSeg prediction pictures and tables")
|
||||
|
||||
if job_type == "mmseg.extract_loss_miou":
|
||||
return CommandSpec(conda_python(conda_env, MY_DIR / "4_4_extract_loss_and_best_miou.py"), MMSEG_DIR, "extract MMSeg loss and best mIoU curves")
|
||||
|
||||
if job_type == "mmseg.delete_epoch":
|
||||
return CommandSpec(conda_python(conda_env, MY_DIR / "3_Find_And_Delete_Special_Epoch.py"), MMSEG_DIR, "find and delete selected epoch checkpoints")
|
||||
|
||||
return None
|
||||
|
||||
2
backend/app/modules/segmodel/__init__.py
Normal file
2
backend/app/modules/segmodel/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
"""SegModel task wrappers."""
|
||||
|
||||
41
backend/app/modules/segmodel/tasks.py
Normal file
41
backend/app/modules/segmodel/tasks.py
Normal file
@@ -0,0 +1,41 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from ...commands import CommandSpec, append_flag, bash, conda_python, required
|
||||
from ...config import settings
|
||||
|
||||
|
||||
SEGMODEL_DIR = settings.source_root / "Seg_All_In_One_SegModel"
|
||||
|
||||
|
||||
def build_segmodel_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
|
||||
env = {"SEG_CONDA_ENV": conda_env}
|
||||
|
||||
if job_type == "segmodel.train":
|
||||
args = conda_python(conda_env, SEGMODEL_DIR / "train.py")
|
||||
append_flag(args, "-a", required(params, "architecture"))
|
||||
return CommandSpec(args, SEGMODEL_DIR, "train one segmentation_models_pytorch architecture")
|
||||
|
||||
if job_type == "segmodel.batch_train":
|
||||
return CommandSpec(bash(SEGMODEL_DIR / "train.sh"), SEGMODEL_DIR, "run legacy SegModel batch training", env=env)
|
||||
|
||||
if job_type == "segmodel.predict":
|
||||
args = conda_python(conda_env, SEGMODEL_DIR / "1_predict.py")
|
||||
append_flag(args, "-a", required(params, "architecture"))
|
||||
choice = str(params.get("run_choice", 1))
|
||||
return CommandSpec(args, SEGMODEL_DIR, "predict with one SegModel run", stdin_text=f"{choice}\n")
|
||||
|
||||
if job_type == "segmodel.batch_predict":
|
||||
return CommandSpec(bash(SEGMODEL_DIR / "predict.sh"), SEGMODEL_DIR, "run legacy SegModel batch prediction", env=env)
|
||||
|
||||
if job_type == "segmodel.flops":
|
||||
script = SEGMODEL_DIR / params.get("script", "2_predict_params_and_FLOPs_V2.py")
|
||||
return CommandSpec(conda_python(conda_env, script), SEGMODEL_DIR, "calculate SegModel params/FLOPs/FPS")
|
||||
|
||||
if job_type == "segmodel.raw_mask_check":
|
||||
return CommandSpec(conda_python(conda_env, SEGMODEL_DIR / "1_predict_raw_masks_check.py"), SEGMODEL_DIR, "check SegModel raw mask completeness")
|
||||
|
||||
if job_type == "segmodel.metrics":
|
||||
return CommandSpec(conda_python(conda_env, SEGMODEL_DIR / "3_predict_matrics_from_log.py"), SEGMODEL_DIR, "parse SegModel training/prediction metrics")
|
||||
|
||||
return None
|
||||
|
||||
2
backend/app/modules/system/__init__.py
Normal file
2
backend/app/modules/system/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
"""System task wrappers."""
|
||||
|
||||
106
backend/app/modules/system/service.py
Normal file
106
backend/app/modules/system/service.py
Normal file
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from ...config import settings
|
||||
|
||||
|
||||
def parse_nvidia_smi_csv(output: str) -> list[dict]:
|
||||
gpus: list[dict] = []
|
||||
for line in output.splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
parts = [part.strip() for part in line.split(",")]
|
||||
if len(parts) < 7:
|
||||
continue
|
||||
index, name, total, used, free, util, temp = parts[:7]
|
||||
try:
|
||||
gpus.append(
|
||||
{
|
||||
"index": int(index),
|
||||
"name": name,
|
||||
"memory_total_mb": int(total),
|
||||
"memory_used_mb": int(used),
|
||||
"memory_free_mb": int(free),
|
||||
"utilization_gpu_percent": int(util),
|
||||
"temperature_c": int(temp),
|
||||
}
|
||||
)
|
||||
except ValueError:
|
||||
continue
|
||||
return gpus
|
||||
|
||||
|
||||
def get_gpus() -> dict:
|
||||
cmd = [
|
||||
"nvidia-smi",
|
||||
"--query-gpu=index,name,memory.total,memory.used,memory.free,utilization.gpu,temperature.gpu",
|
||||
"--format=csv,noheader,nounits",
|
||||
]
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, check=True)
|
||||
return {"available": True, "gpus": parse_nvidia_smi_csv(result.stdout)}
|
||||
except Exception as exc:
|
||||
return {"available": False, "gpus": [], "error": str(exc)}
|
||||
|
||||
|
||||
def get_conda_envs() -> dict:
|
||||
try:
|
||||
result = subprocess.run(["conda", "env", "list"], capture_output=True, text=True, check=True)
|
||||
except Exception as exc:
|
||||
return {"available": False, "envs": [], "error": str(exc)}
|
||||
envs = []
|
||||
for line in result.stdout.splitlines():
|
||||
raw = line.strip()
|
||||
if not raw or raw.startswith("#"):
|
||||
continue
|
||||
marker = "*" in raw.split()
|
||||
parts = raw.replace("*", " ").split()
|
||||
if len(parts) >= 2:
|
||||
envs.append({"name": parts[0], "path": parts[-1], "active": marker})
|
||||
return {"available": True, "envs": envs, "task_default": settings.task_conda_env}
|
||||
|
||||
|
||||
def disk_usage() -> dict:
|
||||
usage = shutil.disk_usage(settings.source_root)
|
||||
return {
|
||||
"path": str(settings.source_root),
|
||||
"total": usage.total,
|
||||
"used": usage.used,
|
||||
"free": usage.free,
|
||||
}
|
||||
|
||||
|
||||
def scan_results() -> list[dict]:
|
||||
roots = [
|
||||
settings.source_root / "DataSet_Public_outputs",
|
||||
settings.source_root / "BestMode_Predict_Results_DataSet_Public",
|
||||
settings.source_root / "Hardisk",
|
||||
settings.source_root / "Seg_All_In_One_Analysis",
|
||||
]
|
||||
exts = {".csv", ".png", ".jpg", ".jpeg", ".svg", ".log", ".pth", ".pt"}
|
||||
results: list[dict] = []
|
||||
for root in roots:
|
||||
if not root.exists():
|
||||
continue
|
||||
for path in root.rglob("*"):
|
||||
if path.is_file() and path.suffix.lower() in exts:
|
||||
try:
|
||||
stat = path.stat()
|
||||
results.append(
|
||||
{
|
||||
"name": path.name,
|
||||
"path": str(path.resolve()),
|
||||
"relative_path": str(path.resolve().relative_to(settings.source_root)),
|
||||
"size": stat.st_size,
|
||||
"modified": stat.st_mtime,
|
||||
"kind": path.suffix.lower().lstrip("."),
|
||||
}
|
||||
)
|
||||
except OSError:
|
||||
continue
|
||||
results.sort(key=lambda item: item["modified"], reverse=True)
|
||||
return results[:1000]
|
||||
|
||||
14
backend/app/modules/system/tasks.py
Normal file
14
backend/app/modules/system/tasks.py
Normal file
@@ -0,0 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from ...commands import CommandSpec, bash
|
||||
from ...config import settings
|
||||
|
||||
|
||||
def build_system_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
|
||||
if job_type == "system.backup":
|
||||
return CommandSpec(bash(settings.source_root / "Back_Up.sh"), settings.source_root, "run legacy backup script")
|
||||
if job_type == "mock.echo":
|
||||
message = params.get("message", "Seg Data Server mock job")
|
||||
return CommandSpec(["python", "-c", f"print({message!r})"], settings.project_root, "test job runner")
|
||||
return None
|
||||
|
||||
2
backend/app/modules/weights/__init__.py
Normal file
2
backend/app/modules/weights/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
"""Weight sync and verification."""
|
||||
|
||||
153
backend/app/modules/weights/service.py
Normal file
153
backend/app/modules/weights/service.py
Normal file
@@ -0,0 +1,153 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from ...config import settings
|
||||
|
||||
WEIGHT_EXTS = {".pt", ".pth", ".onnx", ".engine"}
|
||||
|
||||
|
||||
def sha256_file(path: Path, chunk_size: int = 1024 * 1024 * 8) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while True:
|
||||
chunk = handle.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def iter_source_weights() -> Iterable[Path]:
|
||||
project_root = settings.project_root.resolve()
|
||||
for path in settings.source_root.rglob("*"):
|
||||
if not path.is_file() or path.suffix.lower() not in WEIGHT_EXTS:
|
||||
continue
|
||||
try:
|
||||
path.resolve().relative_to(project_root)
|
||||
continue
|
||||
except ValueError:
|
||||
yield path
|
||||
|
||||
|
||||
def classify_weight(path: Path) -> dict[str, str]:
|
||||
try:
|
||||
rel = str(path.resolve().relative_to(settings.source_root))
|
||||
except ValueError:
|
||||
rel = str(path)
|
||||
lower = rel.lower()
|
||||
if "yolo" in lower:
|
||||
family = "yolo"
|
||||
elif "mmseg" in lower or "my_local_model" in lower:
|
||||
family = "mmseg"
|
||||
elif "segmodel" in lower:
|
||||
family = "segmodel"
|
||||
else:
|
||||
family = "misc"
|
||||
if "best.pt" in lower or "best.pth" in lower:
|
||||
role = "trained_best"
|
||||
elif "last.pt" in lower or "last.pth" in lower:
|
||||
role = "trained_last"
|
||||
elif "pretrain" in lower or "my_local_model" in lower:
|
||||
role = "pretrained"
|
||||
else:
|
||||
role = "weight"
|
||||
return {"family": family, "role": role}
|
||||
|
||||
|
||||
def copy_weight(src: Path, dst: Path, mode: str) -> None:
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
if mode == "hardlink":
|
||||
if dst.exists():
|
||||
dst.unlink()
|
||||
os.link(src, dst)
|
||||
elif mode == "reflink":
|
||||
subprocess.run(["cp", "--reflink=auto", "--preserve=timestamps", str(src), str(dst)], check=True)
|
||||
else:
|
||||
shutil.copy2(src, dst)
|
||||
|
||||
|
||||
def sync_weights(mode: str = "copy", hash_files: bool = True, skip_existing: bool = True) -> dict:
|
||||
files_dir = settings.weights_root / "files"
|
||||
settings.weights_root.mkdir(parents=True, exist_ok=True)
|
||||
entries = []
|
||||
total_bytes = 0
|
||||
for src in sorted(iter_source_weights()):
|
||||
rel = src.resolve().relative_to(settings.source_root)
|
||||
dst = files_dir / rel
|
||||
stat = src.stat()
|
||||
total_bytes += stat.st_size
|
||||
copied = False
|
||||
if not (skip_existing and dst.exists() and dst.stat().st_size == stat.st_size):
|
||||
copy_weight(src, dst, mode)
|
||||
copied = True
|
||||
meta = classify_weight(src)
|
||||
entry = {
|
||||
"source_path": str(rel),
|
||||
"stored_path": str(dst.resolve().relative_to(settings.project_root)),
|
||||
"size": stat.st_size,
|
||||
"family": meta["family"],
|
||||
"role": meta["role"],
|
||||
"copied": copied,
|
||||
}
|
||||
if hash_files:
|
||||
entry["sha256"] = sha256_file(dst)
|
||||
entries.append(entry)
|
||||
|
||||
manifest = {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"source_root": str(settings.source_root),
|
||||
"mode": mode,
|
||||
"count": len(entries),
|
||||
"total_bytes": total_bytes,
|
||||
"files": entries,
|
||||
}
|
||||
manifest_path = settings.weights_root / "manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return manifest
|
||||
|
||||
|
||||
def load_manifest() -> dict:
|
||||
manifest_path = settings.weights_root / "manifest.json"
|
||||
if not manifest_path.exists():
|
||||
return {
|
||||
"generated_at": None,
|
||||
"source_root": str(settings.source_root),
|
||||
"count": 0,
|
||||
"total_bytes": 0,
|
||||
"files": [],
|
||||
}
|
||||
return json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def verify_weights() -> dict:
|
||||
manifest = load_manifest()
|
||||
checked = []
|
||||
ok_count = 0
|
||||
for entry in manifest.get("files", []):
|
||||
path = settings.project_root / entry["stored_path"]
|
||||
exists = path.exists()
|
||||
size_ok = exists and path.stat().st_size == entry.get("size")
|
||||
hash_ok = None
|
||||
if exists and "sha256" in entry:
|
||||
hash_ok = sha256_file(path) == entry["sha256"]
|
||||
ok = bool(exists and size_ok and (hash_ok is not False))
|
||||
ok_count += int(ok)
|
||||
checked.append(
|
||||
{
|
||||
"stored_path": entry["stored_path"],
|
||||
"exists": exists,
|
||||
"size_ok": size_ok,
|
||||
"hash_ok": hash_ok,
|
||||
"ok": ok,
|
||||
}
|
||||
)
|
||||
return {"count": len(checked), "ok_count": ok_count, "items": checked}
|
||||
2
backend/app/modules/yolo/__init__.py
Normal file
2
backend/app/modules/yolo/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
"""YOLO task wrappers."""
|
||||
|
||||
69
backend/app/modules/yolo/tasks.py
Normal file
69
backend/app/modules/yolo/tasks.py
Normal file
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from ...commands import CommandSpec, append_flag, bash, conda_python, required
|
||||
from ...config import settings
|
||||
|
||||
|
||||
YOLO_DIR = settings.source_root / "Seg_All_In_One_YoloModel"
|
||||
VIDEO_YOLO_DIR = settings.source_root / "Seg_Predict_YoloModel"
|
||||
|
||||
|
||||
def build_yolo_task(job_type: str, params: dict, conda_env: str) -> CommandSpec | None:
|
||||
env = {"SEG_CONDA_ENV": conda_env}
|
||||
|
||||
if job_type == "yolo.train":
|
||||
args = conda_python(conda_env, YOLO_DIR / "yolo_train.py")
|
||||
append_flag(args, "--model", required(params, "model"))
|
||||
return CommandSpec(args, YOLO_DIR, "train one Ultralytics YOLO segmentation model")
|
||||
|
||||
if job_type == "yolo.batch_train":
|
||||
return CommandSpec(bash(YOLO_DIR / "yolo_train.sh"), YOLO_DIR, "run legacy YOLO batch training", env=env)
|
||||
|
||||
if job_type == "yolo.predict":
|
||||
args = conda_python(conda_env, YOLO_DIR / "yolo_predict_V2.py")
|
||||
append_flag(args, "--model", required(params, "model"))
|
||||
append_flag(args, "--source", params.get("source"))
|
||||
append_flag(args, "--pt_name", params.get("pt_name", "best.pt"))
|
||||
append_flag(args, "--conf", params.get("conf", 0.2))
|
||||
choice = str(params.get("run_choice", 1))
|
||||
return CommandSpec(args, YOLO_DIR, "predict with one YOLO model", stdin_text=f"{choice}\n")
|
||||
|
||||
if job_type == "yolo.batch_predict":
|
||||
args = bash(YOLO_DIR / "yolo_predict.sh")
|
||||
append_flag(args, "--pt_name", params.get("pt_name", "best.pt"))
|
||||
append_flag(args, "--conf", params.get("conf", 0.2))
|
||||
append_flag(args, "--heatmap_method", params.get("heatmap_method"))
|
||||
return CommandSpec(args, YOLO_DIR, "run legacy YOLO batch prediction", env=env)
|
||||
|
||||
if job_type == "yolo.heatmap":
|
||||
args = conda_python(conda_env, YOLO_DIR / "yolo_predict_visualize_nn.py")
|
||||
append_flag(args, "--model", required(params, "model"))
|
||||
append_flag(args, "--target_layers", params.get("target_layers", "default"))
|
||||
append_flag(args, "--cam_method", params.get("cam_method", "All"))
|
||||
append_flag(args, "--pt_name", params.get("pt_name", "best.pt"))
|
||||
choice = str(params.get("run_choice", 1))
|
||||
return CommandSpec(args, YOLO_DIR, "generate YOLO heatmaps", stdin_text=f"{choice}\n")
|
||||
|
||||
if job_type == "yolo.compare":
|
||||
args = conda_python(conda_env, YOLO_DIR / "yolo_predict_V2_compare_all.py")
|
||||
append_flag(args, "--pt_name", params.get("pt_name", "all"))
|
||||
return CommandSpec(args, YOLO_DIR, "compare all YOLO prediction outputs")
|
||||
|
||||
if job_type == "yolo.raw_mask_check":
|
||||
args = conda_python(conda_env, YOLO_DIR / "yolo_predict_raw_masks_check.py")
|
||||
append_flag(args, "--pt_name", params.get("pt_name", "best.pt"))
|
||||
return CommandSpec(args, YOLO_DIR, "check YOLO raw mask completeness")
|
||||
|
||||
if job_type == "yolo.copy_best":
|
||||
args = bash(YOLO_DIR / "Tool_Yolo_Copy_Best_Model.sh")
|
||||
append_flag(args, "--pt_name", params.get("pt_name", "best.pt"))
|
||||
return CommandSpec(args, YOLO_DIR, "copy YOLO best weights into prediction area")
|
||||
|
||||
if job_type == "yolo.video_visible":
|
||||
return CommandSpec(conda_python(conda_env, VIDEO_YOLO_DIR / "yolo_Seg_Video-V1-Visible.py"), VIDEO_YOLO_DIR, "render visible YOLO video prediction")
|
||||
|
||||
if job_type == "yolo.video_unvisible":
|
||||
return CommandSpec(conda_python(conda_env, VIDEO_YOLO_DIR / "yolo_Seg_Video-V2-UnVisible.py"), VIDEO_YOLO_DIR, "render invisible/headless YOLO video prediction")
|
||||
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user