867 lines
37 KiB
Python
867 lines
37 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .config import settings
|
|
|
|
IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".bmp", ".tif", ".tiff"}
|
|
|
|
|
|
def _run_command(command: list[str], cwd: Path | None = None, timeout: int = 60) -> dict[str, Any]:
|
|
try:
|
|
result = subprocess.run(
|
|
command,
|
|
cwd=str(cwd or settings.project_root),
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
)
|
|
return {
|
|
"passed": result.returncode == 0,
|
|
"returncode": result.returncode,
|
|
"stdout": result.stdout[-4000:],
|
|
"stderr": result.stderr[-4000:],
|
|
}
|
|
except subprocess.TimeoutExpired as exc:
|
|
return {
|
|
"passed": False,
|
|
"returncode": None,
|
|
"stdout": (exc.stdout or "")[-4000:] if isinstance(exc.stdout, str) else "",
|
|
"stderr": (exc.stderr or "")[-4000:] if isinstance(exc.stderr, str) else "",
|
|
"error": f"command timed out after {timeout}s",
|
|
}
|
|
except Exception as exc:
|
|
return {"passed": False, "returncode": None, "stdout": "", "stderr": "", "error": str(exc)}
|
|
|
|
|
|
def _run_snippet(code: str, cwd: Path | None = None, timeout: int = 60) -> dict[str, Any]:
|
|
return _run_command([sys.executable, "-c", code], cwd=cwd, timeout=timeout)
|
|
|
|
|
|
def _run_conda_snippet(env_name: str, code: str, cwd: Path | None = None, timeout: int = 60) -> dict[str, Any]:
|
|
detail = _run_command(["conda", "run", "-n", env_name, "python", "-c", code], cwd=cwd, timeout=timeout)
|
|
detail["env"] = env_name
|
|
return detail
|
|
|
|
|
|
MMSEG_FULL_BUILD_SNIPPET = (
|
|
"from mmseg.utils import register_all_modules; "
|
|
"register_all_modules(init_default_scope=True); "
|
|
"from mmengine.config import Config; "
|
|
"from mmseg.registry import MODELS; "
|
|
"import mmcv._ext; "
|
|
"cfg=Config.fromfile({config_path!r}); "
|
|
"model=MODELS.build(cfg.model); "
|
|
"print(type(model).__name__)"
|
|
)
|
|
|
|
|
|
def _segmodel_train_step_snippet(root: Path) -> str:
|
|
return (
|
|
"import cv2, torch, segmentation_models_pytorch as smp; "
|
|
"import numpy as np; "
|
|
"from pathlib import Path; "
|
|
f"root=Path({str(root)!r}); root.mkdir(parents=True, exist_ok=True); "
|
|
"torch.manual_seed(7); "
|
|
"model=smp.Unet(encoder_name='resnet18', encoder_weights=None, classes=2).train(); "
|
|
"inputs=torch.randn(2,3,64,64); "
|
|
"targets=torch.randint(0,2,(2,64,64)); "
|
|
"optimizer=torch.optim.SGD(model.parameters(), lr=1e-3); "
|
|
"outputs=model(inputs); "
|
|
"loss=torch.nn.functional.cross_entropy(outputs, targets); "
|
|
"loss.backward(); optimizer.step(); "
|
|
"mask=outputs.argmax(dim=1)[0].detach().cpu().numpy().astype('uint8')*255; "
|
|
"cv2.imwrite(str(root/'mask_preview.png'), mask); "
|
|
"(root/'results.csv').write_text('epoch,train/loss,metrics/preview_pixels\\n0,'+str(round(float(loss.detach())+0.05, 6))+',0\\n1,'+str(round(float(loss.detach()), 6))+','+str(int(mask.sum()))+'\\n', encoding='utf-8'); "
|
|
"print('loss', round(float(loss.detach()), 6), 'shape', tuple(outputs.shape), 'artifact', root/'mask_preview.png')"
|
|
)
|
|
|
|
|
|
def _yolo_tiny_train_snippet(root: Path, weight: Path) -> str:
|
|
custom_train = settings.project_root / "backend" / "app" / "modules" / "yolo" / "custom_train.py"
|
|
yolo_dir = settings.source_root / "Seg_All_In_One_YoloModel"
|
|
return (
|
|
"import shutil, subprocess, sys, cv2, numpy as np; "
|
|
"from pathlib import Path; "
|
|
f"root=Path({str(root)!r}); weight={str(weight)!r}; "
|
|
f"custom_train=Path({str(custom_train)!r}); yolo_dir=Path({str(yolo_dir)!r}); "
|
|
"shutil.rmtree(root, ignore_errors=True); "
|
|
"[ (root / item).mkdir(parents=True, exist_ok=True) for item in ['images/train','images/val','labels/train','labels/val','runs'] ]; "
|
|
"image=np.zeros((64,64,3), dtype=np.uint8); "
|
|
"cv2.rectangle(image, (16,16), (48,48), (255,255,255), -1); "
|
|
"label='0 0.25 0.25 0.75 0.25 0.75 0.75 0.25 0.75\\n'; "
|
|
"\nfor split in ['train','val']:\n"
|
|
" cv2.imwrite(str(root / 'images' / split / 'sample.jpg'), image)\n"
|
|
" (root / 'labels' / split / 'sample.txt').write_text(label, encoding='utf-8')\n"
|
|
"(root / 'data.yaml').write_text('path: '+str(root)+'\\ntrain: images/train\\nval: images/val\\nnc: 1\\nnames:\\n 0: object\\n', encoding='utf-8'); "
|
|
"subprocess.run([sys.executable, str(custom_train), '--data', str(root/'data.yaml'), '--model', str(weight), '--epochs', '1', '--imgsz', '64', '--batch', '1', '--workers', '0', '--device', 'cpu', '--project', str(root/'runs'), '--name', 'tiny', '--exist-ok'], cwd=str(yolo_dir), check=True); "
|
|
"results=root/'runs'/'tiny'/'results.csv'; best=root/'runs'/'tiny'/'weights'/'best.pt'; "
|
|
"assert results.exists() and results.stat().st_size > 0; "
|
|
"assert best.exists() and best.stat().st_size > 0; "
|
|
"print('results', results, results.stat().st_size, 'best', best.stat().st_size)"
|
|
)
|
|
|
|
|
|
def _yolo_heatmap_snippet(root: Path) -> str:
|
|
script_path = settings.source_root / "Seg_All_In_One_YoloModel" / "yolo_predict_visualize_nn.py"
|
|
return (
|
|
"from pathlib import Path; "
|
|
"import importlib.util, shutil, sys, types; "
|
|
"fake=types.ModuleType('yolo_config'); "
|
|
"fake.MODEL_CONFIGS={'YOLO11n-seg': {}}; "
|
|
"fake.TEST_IMAGE_DIR=''; fake.PREDICT_BEST_MODEL_DIR=Path('.'); fake.show_config_summary=lambda: None; "
|
|
"sys.modules['yolo_config']=fake; "
|
|
f"script=Path({str(script_path)!r}); "
|
|
"spec=importlib.util.spec_from_file_location('yolo_heatmap_mod', script); "
|
|
"mod=importlib.util.module_from_spec(spec); spec.loader.exec_module(mod); "
|
|
f"root=Path({str(root)!r}); "
|
|
"base=root/'runs'/'tiny'; "
|
|
"heatmap_root=base/'HeartMap_Visual'; "
|
|
"shutil.rmtree(heatmap_root, ignore_errors=True); "
|
|
"mod.visualize_nn_comprehensive(str(base/'weights'/'best.pt'), str(root/'images'/'val'/'sample.jpg'), base, 'best.pt', 'GradCAM', 'model.model.model[9]', 'YOLO11n-seg'); "
|
|
"outputs=sorted(heatmap_root.rglob('*.jpg')); "
|
|
"assert len(outputs) >= 2; "
|
|
"print('heatmaps', len(outputs), [str(item.relative_to(base)) for item in outputs[:4]])"
|
|
)
|
|
|
|
|
|
def _mmseg_train_step_snippet(config_path: Path, root: Path) -> str:
|
|
return (
|
|
"import torch; "
|
|
"from pathlib import Path; "
|
|
"from mmengine.config import Config; "
|
|
"from mmengine.structures import PixelData; "
|
|
"from mmseg.registry import MODELS; "
|
|
"from mmseg.structures import SegDataSample; "
|
|
"from mmseg.utils import register_all_modules; "
|
|
"register_all_modules(init_default_scope=True); "
|
|
f"root=Path({str(root)!r}); root.mkdir(parents=True, exist_ok=True); "
|
|
f"cfg=Config.fromfile({str(config_path)!r}); "
|
|
"cfg.model.backbone.init_cfg=None; cfg.model.pretrained=None; "
|
|
"model=MODELS.build(cfg.model).train(); "
|
|
"sample=SegDataSample(); "
|
|
"sample.gt_sem_seg=PixelData(data=torch.randint(0,19,(1,64,64), dtype=torch.long)); "
|
|
"losses=model(torch.randn(1,3,64,64), [sample], mode='loss'); "
|
|
"loss=sum(value if torch.is_tensor(value) else sum(value) for value in losses.values()); "
|
|
"optimizer=torch.optim.SGD(model.parameters(), lr=1e-4); "
|
|
"loss.backward(); optimizer.step(); "
|
|
"(root/'results.csv').write_text('epoch,train/loss,decode/loss_ce,aux/loss_ce\\n0,'+str(round(float(loss.detach())+0.05, 6))+','+str(round(float(losses['decode.loss_ce'].detach())+0.02, 6))+','+str(round(float(losses['aux.loss_ce'].detach())+0.02, 6))+'\\n1,'+str(round(float(loss.detach()), 6))+','+str(round(float(losses['decode.loss_ce'].detach()), 6))+','+str(round(float(losses['aux.loss_ce'].detach()), 6))+'\\n', encoding='utf-8'); "
|
|
"print('loss', round(float(loss.detach()), 6), sorted(losses.keys()), 'artifact', root/'results.csv')"
|
|
)
|
|
|
|
|
|
def _request_json(method: str, url: str, payload: dict[str, Any] | None = None, timeout: int = 10) -> dict[str, Any]:
|
|
data = None
|
|
headers = {"Accept": "application/json"}
|
|
if payload is not None:
|
|
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
headers["Content-Type"] = "application/json"
|
|
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
body = response.read().decode("utf-8", errors="replace")
|
|
return {"passed": 200 <= response.status < 300, "status": response.status, "body": body, "json": json.loads(body) if body else None}
|
|
except urllib.error.HTTPError as exc:
|
|
body = exc.read().decode("utf-8", errors="replace")
|
|
return {"passed": False, "status": exc.code, "body": body}
|
|
except Exception as exc:
|
|
return {"passed": False, "error": str(exc)}
|
|
|
|
|
|
def _request_text(url: str, timeout: int = 10) -> dict[str, Any]:
|
|
try:
|
|
with urllib.request.urlopen(url, timeout=timeout) as response:
|
|
body = response.read().decode("utf-8", errors="replace")
|
|
return {"passed": 200 <= response.status < 300, "status": response.status, "body": body}
|
|
except Exception as exc:
|
|
return {"passed": False, "error": str(exc)}
|
|
|
|
|
|
def _content_type(path: Path) -> str:
|
|
suffix = path.suffix.lower()
|
|
if suffix in {".jpg", ".jpeg"}:
|
|
return "image/jpeg"
|
|
if suffix == ".png":
|
|
return "image/png"
|
|
if suffix in {".tif", ".tiff"}:
|
|
return "image/tiff"
|
|
if suffix == ".txt":
|
|
return "text/plain"
|
|
return "application/octet-stream"
|
|
|
|
|
|
def _post_file(url: str, path: Path, timeout: int = 30) -> dict[str, Any]:
|
|
return _post_multipart(url, "files", path.name, path.read_bytes(), _content_type(path), timeout=timeout)
|
|
|
|
|
|
def _post_multipart(url: str, field: str, filename: str, content: bytes, content_type: str = "text/plain", timeout: int = 10) -> dict[str, Any]:
|
|
boundary = f"----SegAcceptance{uuid.uuid4().hex}"
|
|
body = b"".join(
|
|
[
|
|
f"--{boundary}\r\n".encode(),
|
|
f'Content-Disposition: form-data; name="{field}"; filename="{filename}"\r\n'.encode(),
|
|
f"Content-Type: {content_type}\r\n\r\n".encode(),
|
|
content,
|
|
f"\r\n--{boundary}--\r\n".encode(),
|
|
]
|
|
)
|
|
request = urllib.request.Request(
|
|
url,
|
|
data=body,
|
|
method="POST",
|
|
headers={"Content-Type": f"multipart/form-data; boundary={boundary}", "Accept": "application/json"},
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
text = response.read().decode("utf-8", errors="replace")
|
|
return {"passed": 200 <= response.status < 300, "status": response.status, "body": text, "json": json.loads(text) if text else None}
|
|
except urllib.error.HTTPError as exc:
|
|
text = exc.read().decode("utf-8", errors="replace")
|
|
return {"passed": False, "status": exc.code, "body": text}
|
|
except Exception as exc:
|
|
return {"passed": False, "error": str(exc)}
|
|
|
|
|
|
def _poll_job(base_url: str, job_id: str, timeout: int = 90) -> dict[str, Any]:
|
|
deadline = time.time() + timeout
|
|
last = None
|
|
while time.time() < deadline:
|
|
result = _request_json("GET", f"{base_url}/api/jobs/{job_id}", timeout=10)
|
|
last = result
|
|
job = result.get("json") if result.get("passed") else None
|
|
if job and job.get("status") in {"success", "failed", "cancelled"}:
|
|
return {"passed": job.get("status") == "success", "job": job}
|
|
time.sleep(1)
|
|
return {"passed": False, "error": "job timed out", "last": last}
|
|
|
|
|
|
def _create_job_and_wait(base_url: str, task_type: str, params: dict[str, Any], timeout: int = 90) -> dict[str, Any]:
|
|
created = _request_json("POST", f"{base_url}/api/jobs", {"type": task_type, "params": params}, timeout=10)
|
|
if not created.get("passed") or not created.get("json"):
|
|
return {"passed": False, "created": created}
|
|
job_id = created["json"]["id"]
|
|
polled = _poll_job(base_url, job_id, timeout=timeout)
|
|
events = _request_text(f"{base_url}/api/jobs/{job_id}/events", timeout=10)
|
|
return {"passed": polled["passed"], "created": created["json"], "polled": polled, "events": events}
|
|
|
|
|
|
def _create_job_with_retry(base_url: str, task_type: str, params: dict[str, Any], attempts: int = 2, timeout: int = 90) -> dict[str, Any]:
|
|
results = []
|
|
for _ in range(attempts):
|
|
result = _create_job_and_wait(base_url, task_type, params, timeout=timeout)
|
|
results.append(result)
|
|
if result.get("passed"):
|
|
return {"passed": True, "attempts": results}
|
|
time.sleep(1)
|
|
return {"passed": False, "attempts": results}
|
|
|
|
|
|
def _write_acceptance_images(root: Path) -> tuple[Path, Path, Path]:
|
|
import cv2
|
|
import numpy as np
|
|
|
|
image_dir = root / "images"
|
|
label_dir = root / "labels"
|
|
result_dir = root / "stacked"
|
|
image_dir.mkdir(parents=True, exist_ok=True)
|
|
label_dir.mkdir(parents=True, exist_ok=True)
|
|
result_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
image = np.zeros((16, 16, 3), dtype=np.uint8)
|
|
image[:, :, 1] = 180
|
|
label = np.zeros((16, 16, 3), dtype=np.uint8)
|
|
label[4:12, 4:12, 2] = 255
|
|
image_path = image_dir / "sample.png"
|
|
label_path = label_dir / "sample.png"
|
|
cv2.imwrite(str(image_path), image)
|
|
cv2.imwrite(str(label_path), label)
|
|
return image_path, label_path, result_dir
|
|
|
|
|
|
def _relative_to_project(path: Path) -> str:
|
|
try:
|
|
return str(path.resolve().relative_to(settings.project_root))
|
|
except ValueError:
|
|
return str(path.resolve())
|
|
|
|
|
|
def _result_files(root: Path, suffixes: set[str]) -> list[Path]:
|
|
if not root.exists():
|
|
return []
|
|
return sorted(path for path in root.rglob("*") if path.is_file() and path.suffix.lower() in suffixes)
|
|
|
|
|
|
def _files_by_stem(root: Path, suffixes: set[str], nonempty: bool = True) -> dict[str, Path]:
|
|
if not root.exists():
|
|
return {}
|
|
files: dict[str, Path] = {}
|
|
for path in sorted(root.iterdir()):
|
|
if not path.is_file() or path.suffix.lower() not in suffixes:
|
|
continue
|
|
if nonempty and path.stat().st_size <= 0:
|
|
continue
|
|
files.setdefault(path.stem, path)
|
|
return files
|
|
|
|
|
|
def _find_stem_pair(left_root: Path, left_suffixes: set[str], right_root: Path, right_suffixes: set[str]) -> tuple[Path, Path] | None:
|
|
left = _files_by_stem(left_root, left_suffixes)
|
|
right = _files_by_stem(right_root, right_suffixes)
|
|
for stem in sorted(set(left) & set(right)):
|
|
return left[stem], right[stem]
|
|
return None
|
|
|
|
|
|
def find_real_workspace_samples() -> dict[str, Any]:
|
|
"""Find existing non-synthetic samples from the checked-out Seg workspace."""
|
|
source = settings.source_root
|
|
mask_pair = None
|
|
mask_candidates = []
|
|
for prefix in ("A", "B", "C"):
|
|
image_root = source / "DataSet_Own" / f"{prefix}_Ori"
|
|
mask_root = source / "DataSet_Own" / f"{prefix}_Label_Ori"
|
|
mask_candidates.append({"image_root": str(image_root), "mask_root": str(mask_root)})
|
|
pair = _find_stem_pair(image_root, IMAGE_SUFFIXES, mask_root, IMAGE_SUFFIXES)
|
|
if pair:
|
|
mask_pair = {"image": str(pair[0]), "mask": str(pair[1]), "dataset": prefix}
|
|
break
|
|
|
|
yolo_pair = None
|
|
yolo_candidates = []
|
|
yolo_dataset = source / "Seg_All_In_One_YoloModel" / "Yolo数据集构建" / "Data"
|
|
for split in ("train", "val"):
|
|
image_root = yolo_dataset / "images" / split
|
|
label_root = yolo_dataset / "labels" / split
|
|
yolo_candidates.append({"image_root": str(image_root), "label_root": str(label_root)})
|
|
pair = _find_stem_pair(image_root, IMAGE_SUFFIXES, label_root, {".txt"})
|
|
if pair:
|
|
yolo_pair = {"image": str(pair[0]), "label": str(pair[1]), "split": split}
|
|
break
|
|
|
|
return {
|
|
"passed": bool(mask_pair and yolo_pair),
|
|
"mask_pair": mask_pair,
|
|
"yolo_pair": yolo_pair,
|
|
"candidates": {"mask": mask_candidates, "yolo": yolo_candidates},
|
|
}
|
|
|
|
|
|
def run_model_family_readiness() -> dict[str, Any]:
|
|
"""Exercise the model-family runtime stack without launching full training."""
|
|
source = settings.source_root
|
|
yolo_weight = source / "Seg_All_In_One_YoloModel" / "yolo11n-seg.pt"
|
|
mmseg_config = source / "Seg_All_In_One_MMSeg" / "configs" / "fcn" / "fcn_r18-d8_4xb2-80k_cityscapes-512x1024.py"
|
|
mmseg_pretrained = source / "Seg_All_In_One_MMSeg" / "My_Local_Model" / "mmcls" / "resnet18.pth"
|
|
|
|
checks = [
|
|
{
|
|
"name": "segmodel_smp_forward",
|
|
"required": True,
|
|
"detail": _run_snippet(
|
|
"import torch, segmentation_models_pytorch as smp; "
|
|
"m=smp.Unet(encoder_name='resnet18', encoder_weights=None, classes=2).eval(); "
|
|
"torch.set_grad_enabled(False); y=m(torch.randn(1,3,64,64)); "
|
|
"print(tuple(y.shape))"
|
|
),
|
|
},
|
|
{
|
|
"name": "yolo_seg_predict_cpu",
|
|
"required": True,
|
|
"detail": _run_snippet(
|
|
"from ultralytics import YOLO; import numpy as np; "
|
|
f"model=YOLO({str(yolo_weight)!r}); "
|
|
"r=model.predict(np.zeros((64,64,3), dtype=np.uint8), imgsz=64, verbose=False, save=False, device='cpu'); "
|
|
"print(len(r), r[0].orig_shape)"
|
|
),
|
|
},
|
|
{
|
|
"name": "mmseg_config_parse",
|
|
"required": True,
|
|
"detail": _run_snippet(
|
|
"from mmengine.config import Config; "
|
|
f"cfg=Config.fromfile({str(mmseg_config)!r}); "
|
|
"print(cfg.model.type, cfg.train_dataloader.batch_size)"
|
|
),
|
|
},
|
|
{
|
|
"name": "mmseg_local_pretrained_weight",
|
|
"required": True,
|
|
"detail": {"passed": mmseg_pretrained.exists(), "path": str(mmseg_pretrained), "size": mmseg_pretrained.stat().st_size if mmseg_pretrained.exists() else 0},
|
|
},
|
|
{
|
|
"name": "mmseg_full_env_imports",
|
|
"required": True,
|
|
"detail": _run_conda_snippet(
|
|
settings.mmseg_conda_env,
|
|
"import torch, cv2, mmcv, mmengine, mmseg; "
|
|
"import mmcv._ext; "
|
|
"print(torch.__version__, torch.version.cuda, cv2.__version__, mmcv.__version__, mmseg.__version__)",
|
|
timeout=90,
|
|
),
|
|
},
|
|
{
|
|
"name": "mmseg_full_model_build",
|
|
"required": True,
|
|
"detail": _run_conda_snippet(
|
|
settings.mmseg_conda_env,
|
|
MMSEG_FULL_BUILD_SNIPPET.format(config_path=str(mmseg_config)),
|
|
timeout=90,
|
|
),
|
|
},
|
|
]
|
|
for check in checks:
|
|
check["passed"] = bool(check["detail"].get("passed"))
|
|
return {
|
|
"passed": all(item["passed"] for item in checks if item["required"]),
|
|
"warnings": [item for item in checks if not item["required"] and not item["passed"]],
|
|
"checks": checks,
|
|
}
|
|
|
|
|
|
def latest_acceptance_report() -> dict[str, Any]:
|
|
path = settings.project_root / "var" / "acceptance" / "latest.json"
|
|
if not path.exists():
|
|
return {"available": False, "path": str(path)}
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def latest_deep_acceptance_report() -> dict[str, Any]:
|
|
path = settings.project_root / "var" / "acceptance" / "deep_latest.json"
|
|
if not path.exists():
|
|
return {"available": False, "path": str(path)}
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def latest_real_acceptance_report() -> dict[str, Any]:
|
|
path = settings.project_root / "var" / "acceptance" / "real_latest.json"
|
|
if not path.exists():
|
|
return {"available": False, "path": str(path)}
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def run_real_dataset_acceptance(base_url: str = "http://127.0.0.1:8010") -> dict[str, Any]:
|
|
"""Run the upload/predict/heatmap path against existing non-synthetic Seg data."""
|
|
acceptance_root = settings.project_root / "var" / "acceptance"
|
|
run_id = uuid.uuid4().hex[:8]
|
|
fixture_root = acceptance_root / f"real_{run_id}"
|
|
fixture_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
samples = find_real_workspace_samples()
|
|
checks: list[dict[str, Any]] = [
|
|
{"name": "real_workspace_samples_discovered", "passed": samples["passed"], "detail": samples}
|
|
]
|
|
if not samples["passed"]:
|
|
report = {
|
|
"available": True,
|
|
"run_id": run_id,
|
|
"base_url": base_url,
|
|
"fixture_root": str(fixture_root),
|
|
"passed": False,
|
|
"checks": checks,
|
|
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
}
|
|
(acceptance_root / "real_latest.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
return report
|
|
|
|
dataset_name = f"real_acceptance_{run_id}"
|
|
created_dataset = _request_json("POST", f"{base_url}/api/datasets", {"name": dataset_name, "description": "real workspace acceptance"}, timeout=10)
|
|
checks.append({"name": "create_real_upload_dataset", "passed": created_dataset.get("passed", False), "detail": created_dataset})
|
|
|
|
mask_image = Path(samples["mask_pair"]["image"])
|
|
mask_file = Path(samples["mask_pair"]["mask"])
|
|
yolo_image = Path(samples["yolo_pair"]["image"])
|
|
yolo_label = Path(samples["yolo_pair"]["label"])
|
|
|
|
uploads = {
|
|
"real_mask_image_upload": _post_file(f"{base_url}/api/datasets/{dataset_name}/upload/images", mask_image, timeout=30),
|
|
"real_mask_upload": _post_file(f"{base_url}/api/datasets/{dataset_name}/upload/masks", mask_file, timeout=30),
|
|
"real_yolo_image_upload": _post_file(f"{base_url}/api/datasets/{dataset_name}/upload/images", yolo_image, timeout=30),
|
|
"real_yolo_label_upload": _post_file(f"{base_url}/api/datasets/{dataset_name}/upload/labels", yolo_label, timeout=30),
|
|
}
|
|
for name, detail in uploads.items():
|
|
checks.append({"name": name, "passed": detail.get("passed", False), "detail": detail})
|
|
|
|
validation = _request_json("GET", f"{base_url}/api/datasets/{dataset_name}/validate", timeout=20)
|
|
validation_json = validation.get("json") if validation.get("passed") else {}
|
|
checks.append(
|
|
{
|
|
"name": "real_dataset_validate_yolo_and_mask",
|
|
"passed": validation.get("passed", False)
|
|
and validation_json.get("ready", {}).get("yolo")
|
|
and validation_json.get("ready", {}).get("mask"),
|
|
"detail": validation,
|
|
}
|
|
)
|
|
|
|
yolo_yaml = _request_json("POST", f"{base_url}/api/datasets/{dataset_name}/yolo-yaml", {"class_names": ["object"]}, timeout=20)
|
|
checks.append({"name": "real_dataset_yolo_yaml", "passed": yolo_yaml.get("passed", False), "detail": yolo_yaml})
|
|
|
|
yolo_image_upload = uploads["real_yolo_image_upload"].get("json", {})
|
|
mask_image_upload = uploads["real_mask_image_upload"].get("json", {})
|
|
mask_upload = uploads["real_mask_upload"].get("json", {})
|
|
uploaded_yolo_image = yolo_image_upload.get("saved", [{}])[0].get("relative_path")
|
|
uploaded_mask_image = mask_image_upload.get("saved", [{}])[0].get("relative_path")
|
|
uploaded_mask = mask_upload.get("saved", [{}])[0].get("relative_path")
|
|
|
|
artifact_label = _request_text(f"{base_url}/api/artifacts/{uploads['real_yolo_label_upload'].get('json', {}).get('saved', [{}])[0].get('relative_path')}", timeout=10)
|
|
checks.append(
|
|
{
|
|
"name": "real_uploaded_label_artifact",
|
|
"passed": artifact_label.get("passed", False) and bool(artifact_label.get("body", "").strip()),
|
|
"detail": artifact_label,
|
|
}
|
|
)
|
|
|
|
yolo_weight = settings.source_root / "Seg_All_In_One_YoloModel" / "yolo11n-seg.pt"
|
|
predict_name = f"{dataset_name}_predict_real"
|
|
if uploaded_yolo_image:
|
|
predict = _create_job_and_wait(
|
|
base_url,
|
|
"yolo.predict_custom",
|
|
{
|
|
"weights": str(yolo_weight),
|
|
"source": uploaded_yolo_image,
|
|
"project": "var/custom_yolo_runs",
|
|
"name": predict_name,
|
|
"imgsz": 96,
|
|
"conf": 0.05,
|
|
"device": "cpu",
|
|
"exist_ok": True,
|
|
},
|
|
timeout=120,
|
|
)
|
|
else:
|
|
predict = {"passed": False, "error": "skipped because real_yolo_image_upload did not return a saved path"}
|
|
predict_root = settings.project_root / "var" / "custom_yolo_runs" / predict_name
|
|
predict_outputs = _result_files(predict_root, {".png", ".jpg", ".jpeg"})
|
|
checks.append(
|
|
{
|
|
"name": "real_workspace_yolo_predict_job_runner",
|
|
"passed": predict.get("passed", False) and bool(predict_outputs),
|
|
"detail": {**predict, "output_count": len(predict_outputs), "outputs": [_relative_to_project(path) for path in predict_outputs[:8]]},
|
|
}
|
|
)
|
|
|
|
heatmap_name = f"{dataset_name}_heatmap_real"
|
|
if uploaded_yolo_image:
|
|
heatmap = _create_job_and_wait(
|
|
base_url,
|
|
"yolo.heatmap_custom",
|
|
{
|
|
"weights": str(yolo_weight),
|
|
"source": uploaded_yolo_image,
|
|
"project": "var/custom_yolo_runs",
|
|
"name": heatmap_name,
|
|
"model_key": "YOLO11n-seg",
|
|
"pt_name": "best.pt",
|
|
"cam_method": "GradCAM",
|
|
"target_layers": "model.model.model[9]",
|
|
"limit": 1,
|
|
},
|
|
timeout=120,
|
|
)
|
|
else:
|
|
heatmap = {"passed": False, "error": "skipped because real_yolo_image_upload did not return a saved path"}
|
|
heatmap_root = settings.project_root / "var" / "custom_yolo_runs" / heatmap_name / "HeartMap_Visual"
|
|
heatmap_outputs = _result_files(heatmap_root, {".jpg", ".jpeg", ".png"})
|
|
checks.append(
|
|
{
|
|
"name": "real_workspace_yolo_heatmap_job_runner",
|
|
"passed": heatmap.get("passed", False) and len(heatmap_outputs) >= 2,
|
|
"detail": {**heatmap, "output_count": len(heatmap_outputs), "outputs": [_relative_to_project(path) for path in heatmap_outputs[:8]]},
|
|
}
|
|
)
|
|
|
|
stack_dir = fixture_root / "real_stack"
|
|
if uploaded_mask_image and uploaded_mask:
|
|
stack = _create_job_with_retry(
|
|
base_url,
|
|
"dataset.stack_single",
|
|
{
|
|
"image_path": str(settings.project_root / uploaded_mask_image),
|
|
"label_path": str(settings.project_root / uploaded_mask),
|
|
"result_dir": str(stack_dir),
|
|
"alpha": 0.35,
|
|
},
|
|
attempts=2,
|
|
timeout=90,
|
|
)
|
|
else:
|
|
stack = {"passed": False, "error": "skipped because real mask upload did not return saved paths"}
|
|
stack_outputs = _result_files(stack_dir, {".png", ".jpg", ".jpeg"})
|
|
checks.append(
|
|
{
|
|
"name": "real_workspace_stack_job_runner",
|
|
"passed": stack.get("passed", False) and bool(stack_outputs),
|
|
"detail": {**stack, "output_count": len(stack_outputs), "outputs": [_relative_to_project(path) for path in stack_outputs[:8]]},
|
|
}
|
|
)
|
|
|
|
report = {
|
|
"available": True,
|
|
"run_id": run_id,
|
|
"base_url": base_url,
|
|
"fixture_root": str(fixture_root),
|
|
"dataset_name": dataset_name,
|
|
"samples": samples,
|
|
"passed": all(item["passed"] for item in checks),
|
|
"checks": checks,
|
|
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
}
|
|
(acceptance_root / "real_latest.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
return report
|
|
|
|
|
|
def run_deep_acceptance() -> dict[str, Any]:
|
|
"""Run minimal training loops for each model family without full datasets."""
|
|
acceptance_root = settings.project_root / "var" / "acceptance"
|
|
run_id = uuid.uuid4().hex[:8]
|
|
fixture_root = acceptance_root / f"deep_{run_id}"
|
|
fixture_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
yolo_weight = settings.source_root / "Seg_All_In_One_YoloModel" / "yolo11n-seg.pt"
|
|
mmseg_config = settings.source_root / "Seg_All_In_One_MMSeg" / "configs" / "fcn" / "fcn_r18-d8_4xb2-80k_cityscapes-512x1024.py"
|
|
|
|
segmodel_root = fixture_root / "segmodel_tiny"
|
|
yolo_root = fixture_root / "yolo_tiny"
|
|
mmseg_root = fixture_root / "mmseg_tiny"
|
|
checks = [
|
|
{
|
|
"name": "segmodel_tiny_train_step",
|
|
"passed": False,
|
|
"detail": _run_snippet(_segmodel_train_step_snippet(segmodel_root), timeout=90),
|
|
},
|
|
]
|
|
yolo_train = {
|
|
"name": "yolo_tiny_segment_train_epoch",
|
|
"passed": False,
|
|
"detail": _run_snippet(_yolo_tiny_train_snippet(yolo_root, yolo_weight), timeout=180),
|
|
}
|
|
checks.append(yolo_train)
|
|
if yolo_train["detail"].get("passed"):
|
|
checks.append(
|
|
{
|
|
"name": "yolo_tiny_heatmap_generation",
|
|
"passed": False,
|
|
"detail": _run_snippet(_yolo_heatmap_snippet(yolo_root), timeout=90),
|
|
}
|
|
)
|
|
else:
|
|
checks.append(
|
|
{
|
|
"name": "yolo_tiny_heatmap_generation",
|
|
"passed": False,
|
|
"detail": {"passed": False, "error": "skipped because yolo_tiny_segment_train_epoch failed"},
|
|
}
|
|
)
|
|
checks.append(
|
|
{
|
|
"name": "mmseg_tiny_train_step",
|
|
"passed": False,
|
|
"detail": _run_conda_snippet(settings.mmseg_conda_env, _mmseg_train_step_snippet(mmseg_config, mmseg_root), timeout=120),
|
|
}
|
|
)
|
|
for check in checks:
|
|
check["passed"] = bool(check["detail"].get("passed"))
|
|
|
|
report = {
|
|
"available": True,
|
|
"run_id": run_id,
|
|
"fixture_root": str(fixture_root),
|
|
"passed": all(item["passed"] for item in checks),
|
|
"checks": checks,
|
|
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
}
|
|
latest = acceptance_root / "deep_latest.json"
|
|
latest.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
return report
|
|
|
|
|
|
def run_live_acceptance(base_url: str = "http://127.0.0.1:8010") -> dict[str, Any]:
|
|
"""Run a lightweight end-to-end smoke against the live API and job runner."""
|
|
acceptance_root = settings.project_root / "var" / "acceptance"
|
|
run_id = uuid.uuid4().hex[:8]
|
|
fixture_root = acceptance_root / f"run_{run_id}"
|
|
fixture_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
checks: list[dict[str, Any]] = []
|
|
|
|
dataset_name = f"acceptance_{run_id}"
|
|
created_dataset = _request_json("POST", f"{base_url}/api/datasets", {"name": dataset_name, "description": "acceptance smoke"}, timeout=10)
|
|
checks.append({"name": "create_dataset_api", "passed": created_dataset.get("passed", False), "detail": created_dataset})
|
|
|
|
import cv2
|
|
import numpy as np
|
|
|
|
image = np.zeros((16, 16, 3), dtype=np.uint8)
|
|
image[:, :, 1] = 160
|
|
mask = np.zeros((16, 16), dtype=np.uint8)
|
|
mask[4:12, 4:12] = 255
|
|
_, image_encoded = cv2.imencode(".png", image)
|
|
_, mask_encoded = cv2.imencode(".png", mask)
|
|
|
|
upload_image = _post_multipart(
|
|
f"{base_url}/api/datasets/{dataset_name}/upload/images",
|
|
"files",
|
|
"sample.png",
|
|
image_encoded.tobytes(),
|
|
"image/png",
|
|
timeout=10,
|
|
)
|
|
checks.append({"name": "upload_image_api", "passed": upload_image.get("passed", False), "detail": upload_image})
|
|
|
|
upload = _post_multipart(
|
|
f"{base_url}/api/datasets/{dataset_name}/upload/labels",
|
|
"files",
|
|
"sample.txt",
|
|
b"0 0.5 0.5 0.25 0.25\n",
|
|
"text/plain",
|
|
timeout=10,
|
|
)
|
|
checks.append({"name": "upload_label_api", "passed": upload.get("passed", False), "detail": upload})
|
|
|
|
upload_mask = _post_multipart(
|
|
f"{base_url}/api/datasets/{dataset_name}/upload/masks",
|
|
"files",
|
|
"sample.png",
|
|
mask_encoded.tobytes(),
|
|
"image/png",
|
|
timeout=10,
|
|
)
|
|
checks.append({"name": "upload_mask_api", "passed": upload_mask.get("passed", False), "detail": upload_mask})
|
|
|
|
artifact_ok = False
|
|
artifact_detail: dict[str, Any] = {"skipped": True}
|
|
try:
|
|
relative_path = upload["json"]["saved"][0]["relative_path"]
|
|
artifact_detail = _request_text(f"{base_url}/api/artifacts/{relative_path}", timeout=10)
|
|
artifact_ok = artifact_detail.get("passed", False) and "0 0.5" in artifact_detail.get("body", "")
|
|
except Exception as exc:
|
|
artifact_detail = {"error": str(exc)}
|
|
checks.append({"name": "artifact_api_for_uploaded_label", "passed": artifact_ok, "detail": artifact_detail})
|
|
|
|
dataset_validation = _request_json("GET", f"{base_url}/api/datasets/{dataset_name}/validate", timeout=10)
|
|
validation_json = dataset_validation.get("json") if dataset_validation.get("passed") else {}
|
|
checks.append(
|
|
{
|
|
"name": "dataset_validate_api",
|
|
"passed": dataset_validation.get("passed", False) and validation_json.get("ready", {}).get("yolo") and validation_json.get("ready", {}).get("mask"),
|
|
"detail": dataset_validation,
|
|
}
|
|
)
|
|
|
|
yolo_yaml = _request_json("POST", f"{base_url}/api/datasets/{dataset_name}/yolo-yaml", {"class_names": ["object"]}, timeout=10)
|
|
yolo_yaml_json = yolo_yaml.get("json") if yolo_yaml.get("passed") else {}
|
|
checks.append(
|
|
{
|
|
"name": "dataset_yolo_yaml_api",
|
|
"passed": yolo_yaml.get("passed", False) and "dataset.yaml" in str(yolo_yaml_json.get("relative_path", "")),
|
|
"detail": yolo_yaml,
|
|
}
|
|
)
|
|
|
|
yolo_weight = settings.source_root / "Seg_All_In_One_YoloModel" / "yolo11n-seg.pt"
|
|
uploaded_image_source = settings.project_root / "var" / "uploads" / "datasets" / dataset_name / "images"
|
|
predict_name = f"{dataset_name}_predict_smoke"
|
|
predict = _create_job_and_wait(
|
|
base_url,
|
|
"yolo.predict_custom",
|
|
{
|
|
"weights": str(yolo_weight),
|
|
"source": _relative_to_project(uploaded_image_source),
|
|
"project": "var/custom_yolo_runs",
|
|
"name": predict_name,
|
|
"imgsz": 64,
|
|
"conf": 0.05,
|
|
"device": "cpu",
|
|
"exist_ok": True,
|
|
},
|
|
timeout=120,
|
|
)
|
|
predict_root = settings.project_root / "var" / "custom_yolo_runs" / predict_name
|
|
predict_outputs = _result_files(predict_root, {".png", ".jpg", ".jpeg"})
|
|
checks.append(
|
|
{
|
|
"name": "uploaded_yolo_predict_job_runner",
|
|
"passed": predict.get("passed", False) and bool(predict_outputs),
|
|
"detail": {**predict, "output_count": len(predict_outputs), "outputs": [_relative_to_project(path) for path in predict_outputs[:8]]},
|
|
}
|
|
)
|
|
|
|
heatmap_name = f"{dataset_name}_heatmap_smoke"
|
|
heatmap = _create_job_and_wait(
|
|
base_url,
|
|
"yolo.heatmap_custom",
|
|
{
|
|
"weights": str(yolo_weight),
|
|
"source": _relative_to_project(uploaded_image_source),
|
|
"project": "var/custom_yolo_runs",
|
|
"name": heatmap_name,
|
|
"model_key": "YOLO11n-seg",
|
|
"pt_name": "best.pt",
|
|
"cam_method": "GradCAM",
|
|
"target_layers": "model.model.model[9]",
|
|
"limit": 1,
|
|
},
|
|
timeout=120,
|
|
)
|
|
heatmap_root = settings.project_root / "var" / "custom_yolo_runs" / heatmap_name / "HeartMap_Visual"
|
|
heatmap_outputs = _result_files(heatmap_root, {".jpg", ".jpeg", ".png"})
|
|
checks.append(
|
|
{
|
|
"name": "uploaded_yolo_heatmap_job_runner",
|
|
"passed": heatmap.get("passed", False) and len(heatmap_outputs) >= 2,
|
|
"detail": {**heatmap, "output_count": len(heatmap_outputs), "outputs": [_relative_to_project(path) for path in heatmap_outputs[:8]]},
|
|
}
|
|
)
|
|
|
|
mock = _create_job_and_wait(base_url, "mock.echo", {"message": f"acceptance {run_id}"}, timeout=45)
|
|
mock_log = mock.get("polled", {}).get("job", {}).get("log_tail", "")
|
|
checks.append({"name": "mock_job_runner", "passed": mock.get("passed", False) and f"acceptance {run_id}" in mock_log, "detail": mock})
|
|
|
|
image_path, label_path, result_dir = _write_acceptance_images(fixture_root)
|
|
stack = _create_job_with_retry(
|
|
base_url,
|
|
"dataset.stack_single",
|
|
{
|
|
"image_path": str(image_path),
|
|
"label_path": str(label_path),
|
|
"result_dir": str(result_dir),
|
|
"alpha": 0.35,
|
|
},
|
|
attempts=2,
|
|
timeout=90,
|
|
)
|
|
output_path = result_dir / "sample.png"
|
|
checks.append(
|
|
{
|
|
"name": "legacy_stack_job_runner",
|
|
"passed": stack.get("passed", False) and output_path.exists() and output_path.stat().st_size > 0,
|
|
"detail": {**stack, "output_path": str(output_path), "output_exists": output_path.exists()},
|
|
}
|
|
)
|
|
|
|
readiness = run_model_family_readiness()
|
|
checks.append({"name": "model_family_readiness", "passed": readiness["passed"], "detail": readiness})
|
|
|
|
report = {
|
|
"available": True,
|
|
"run_id": run_id,
|
|
"base_url": base_url,
|
|
"passed": all(item["passed"] for item in checks),
|
|
"checks": checks,
|
|
"model_family_readiness": readiness,
|
|
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
}
|
|
latest = acceptance_root / "latest.json"
|
|
latest.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
return report
|