Add dataset QA and custom YOLO training flow

backend/app/acceptance.py

@@ -77,11 +77,13 @@ SEGMODEL_TRAIN_STEP_SNIPPET = (
 
 
 def _yolo_tiny_train_snippet(root: Path, weight: Path) -> str:
+    custom_train = settings.project_root / "backend" / "app" / "modules" / "yolo" / "custom_train.py"
+    yolo_dir = settings.source_root / "Seg_All_In_One_YoloModel"
     return (
-        "import shutil, cv2, numpy as np; "
+        "import shutil, subprocess, sys, cv2, numpy as np; "
         "from pathlib import Path; "
-        "from ultralytics import YOLO; "
         f"root=Path({str(root)!r}); weight={str(weight)!r}; "
+        f"custom_train=Path({str(custom_train)!r}); yolo_dir=Path({str(yolo_dir)!r}); "
         "shutil.rmtree(root, ignore_errors=True); "
         "[ (root / item).mkdir(parents=True, exist_ok=True) for item in ['images/train','images/val','labels/train','labels/val','runs'] ]; "
         "image=np.zeros((64,64,3), dtype=np.uint8); "
@@ -91,8 +93,7 @@ def _yolo_tiny_train_snippet(root: Path, weight: Path) -> str:
         "    cv2.imwrite(str(root / 'images' / split / 'sample.jpg'), image)\n"
         "    (root / 'labels' / split / 'sample.txt').write_text(label, encoding='utf-8')\n"
         "(root / 'data.yaml').write_text('path: '+str(root)+'\\ntrain: images/train\\nval: images/val\\nnc: 1\\nnames:\\n  0: object\\n', encoding='utf-8'); "
-        "model=YOLO(weight); "
-        "model.train(data=str(root/'data.yaml'), epochs=1, imgsz=64, batch=1, workers=0, device='cpu', project=str(root/'runs'), name='tiny', exist_ok=True, verbose=False, plots=False, val=False); "
+        "subprocess.run([sys.executable, str(custom_train), '--data', str(root/'data.yaml'), '--model', str(weight), '--epochs', '1', '--imgsz', '64', '--batch', '1', '--workers', '0', '--device', 'cpu', '--project', str(root/'runs'), '--name', 'tiny', '--exist-ok'], cwd=str(yolo_dir), check=True); "
         "results=root/'runs'/'tiny'/'results.csv'; best=root/'runs'/'tiny'/'weights'/'best.pt'; "
         "assert results.exists() and results.stat().st_size > 0; "
         "assert best.exists() and best.stat().st_size > 0; "
@@ -418,16 +419,46 @@ def run_live_acceptance(base_url: str = "http://127.0.0.1:8010") -> dict[str, An
     created_dataset = _request_json("POST", f"{base_url}/api/datasets", {"name": dataset_name, "description": "acceptance smoke"}, timeout=10)
     checks.append({"name": "create_dataset_api", "passed": created_dataset.get("passed", False), "detail": created_dataset})
 
+    import cv2
+    import numpy as np
+
+    image = np.zeros((16, 16, 3), dtype=np.uint8)
+    image[:, :, 1] = 160
+    mask = np.zeros((16, 16), dtype=np.uint8)
+    mask[4:12, 4:12] = 255
+    _, image_encoded = cv2.imencode(".png", image)
+    _, mask_encoded = cv2.imencode(".png", mask)
+
+    upload_image = _post_multipart(
+        f"{base_url}/api/datasets/{dataset_name}/upload/images",
+        "files",
+        "sample.png",
+        image_encoded.tobytes(),
+        "image/png",
+        timeout=10,
+    )
+    checks.append({"name": "upload_image_api", "passed": upload_image.get("passed", False), "detail": upload_image})
+
     upload = _post_multipart(
         f"{base_url}/api/datasets/{dataset_name}/upload/labels",
         "files",
-        "label 01.txt",
+        "sample.txt",
         b"0 0.5 0.5 0.25 0.25\n",
         "text/plain",
         timeout=10,
     )
     checks.append({"name": "upload_label_api", "passed": upload.get("passed", False), "detail": upload})
 
+    upload_mask = _post_multipart(
+        f"{base_url}/api/datasets/{dataset_name}/upload/masks",
+        "files",
+        "sample.png",
+        mask_encoded.tobytes(),
+        "image/png",
+        timeout=10,
+    )
+    checks.append({"name": "upload_mask_api", "passed": upload_mask.get("passed", False), "detail": upload_mask})
+
     artifact_ok = False
     artifact_detail: dict[str, Any] = {"skipped": True}
     try:
@@ -438,6 +469,26 @@ def run_live_acceptance(base_url: str = "http://127.0.0.1:8010") -> dict[str, An
         artifact_detail = {"error": str(exc)}
     checks.append({"name": "artifact_api_for_uploaded_label", "passed": artifact_ok, "detail": artifact_detail})
 
+    dataset_validation = _request_json("GET", f"{base_url}/api/datasets/{dataset_name}/validate", timeout=10)
+    validation_json = dataset_validation.get("json") if dataset_validation.get("passed") else {}
+    checks.append(
+        {
+            "name": "dataset_validate_api",
+            "passed": dataset_validation.get("passed", False) and validation_json.get("ready", {}).get("yolo") and validation_json.get("ready", {}).get("mask"),
+            "detail": dataset_validation,
+        }
+    )
+
+    yolo_yaml = _request_json("POST", f"{base_url}/api/datasets/{dataset_name}/yolo-yaml", {"class_names": ["object"]}, timeout=10)
+    yolo_yaml_json = yolo_yaml.get("json") if yolo_yaml.get("passed") else {}
+    checks.append(
+        {
+            "name": "dataset_yolo_yaml_api",
+            "passed": yolo_yaml.get("passed", False) and "dataset.yaml" in str(yolo_yaml_json.get("relative_path", "")),
+            "detail": yolo_yaml,
+        }
+    )
+
     mock = _create_job_and_wait(base_url, "mock.echo", {"message": f"acceptance {run_id}"}, timeout=45)
     mock_log = mock.get("polled", {}).get("job", {}).get("log_tail", "")
     checks.append({"name": "mock_job_runner", "passed": mock.get("passed", False) and f"acceptance {run_id}" in mock_log, "detail": mock})