Add runtime environment readiness checks

backend/app/agents/evaluation_agent.py

@@ -43,9 +43,13 @@ def evaluate_project() -> dict:
         "dataset_quality_ui": "DatasetQuality" in frontend_text and "generateSelectedYoloYaml" in frontend_text,
         "loss_result_ui": "loss" in frontend_text.lower() and "heatmap" in frontend_text.lower() and "CurvePanel" in frontend_text,
         "job_progress_ui": "JobProgressBar" in frontend_text and "progressTrack" in frontend_text,
+        "runtime_readiness_ui": "runtimeReadiness" in frontend_text and "环境就绪" in frontend_text,
         "dataset_api": "/api/datasets" in backend_text and "api_upload_dataset_files" in backend_text,
         "dataset_quality_api": "/api/datasets/{dataset_name}/validate" in backend_text and "/api/datasets/{dataset_name}/yolo-yaml" in backend_text,
         "job_progress_api": "progress_from_log_path" in backend_text and '"progress"' in backend_text,
+        "runtime_readiness_api": "/api/system/readiness" in backend_text,
+        "runtime_bootstrap_scripts": (settings.project_root / "scripts" / "bootstrap_conda_envs.sh").exists()
+        and (settings.project_root / "scripts" / "verify_runtime_envs.py").exists(),
         "curve_api": "/api/results/curves" in backend_text,
         "deep_acceptance_api": "/api/acceptance/deep" in backend_text,
         "deep_acceptance_ui": "runDeepAcceptance" in frontend_text and "深度训练" in frontend_text,

backend/app/agents/validation_agent.py

@@ -13,7 +13,7 @@ from ..catalog import get_catalog
 from ..config import settings
 from ..coverage import get_coverage_report
 from ..modules.results.service import scan_training_curves
-from ..modules.system.service import get_conda_envs, get_gpus
+from ..modules.system.service import get_conda_envs, get_gpus, get_runtime_readiness
 from ..modules.weights.service import load_manifest
 from ..progress import parse_progress
 
@@ -64,6 +64,8 @@ def validate_project(run_build: bool = False) -> dict:
     env_names = [item["name"] for item in get_conda_envs().get("envs", [])]
     checks.append({"name": "task_env_exists", "passed": settings.task_conda_env in env_names, "detail": {"env": settings.task_conda_env}})
     checks.append({"name": "mmseg_env_exists", "passed": settings.mmseg_conda_env in env_names, "detail": {"env": settings.mmseg_conda_env}})
+    runtime_readiness = get_runtime_readiness(force=True)
+    checks.append({"name": "runtime_env_readiness", "passed": runtime_readiness["passed"], "detail": runtime_readiness})
 
     smoke = _run(
         [
@@ -102,6 +104,7 @@ def validate_project(run_build: bool = False) -> dict:
         health = _fetch(f"{backend_url}/api/health")
         datasets = _fetch(f"{backend_url}/api/datasets")
         live_jobs = _fetch(f"{backend_url}/api/jobs")
+        live_readiness = _fetch(f"{backend_url}/api/system/readiness")
         live_coverage = _fetch(f"{backend_url}/api/coverage")
         live_curves = _fetch(f"{backend_url}/api/results/curves")
         frontend = _fetch(frontend_url)
@@ -116,6 +119,11 @@ def validate_project(run_build: bool = False) -> dict:
             "passed": live_jobs["passed"] and isinstance(live_job_items, list) and (not live_job_items or "progress" in live_job_items[0]),
             "detail": live_jobs,
         })
+        checks.append({
+            "name": "live_runtime_readiness_api",
+            "passed": live_readiness["passed"] and '"passed":true' in live_readiness.get("body", "").replace(" ", ""),
+            "detail": live_readiness,
+        })
         checks.append({"name": "live_coverage_api", "passed": live_coverage["passed"] and '"task_build_passed":true' in live_coverage.get("body", "").replace(" ", ""), "detail": live_coverage})
         checks.append({"name": "live_training_curves_api", "passed": live_curves["passed"] and live_curves.get("body", "").lstrip().startswith("["), "detail": live_curves})
         checks.append({"name": "live_frontend_index", "passed": frontend["passed"] and "Seg Data Server" in frontend.get("body", ""), "detail": frontend})

backend/app/main.py

@@ -15,7 +15,7 @@ from .config import settings
 from .coverage import get_coverage_report
 from .jobs import cancel_job, create_job
 from .modules.results.service import scan_results, scan_training_curves
-from .modules.system.service import disk_usage, get_conda_envs, get_gpus
+from .modules.system.service import disk_usage, get_conda_envs, get_gpus, get_runtime_readiness
 from .modules.dataset.service import create_dataset, generate_yolo_dataset_yaml, list_uploaded_datasets, save_upload, validate_dataset
 from .modules.weights.service import load_manifest, sync_weights, verify_weights
 from .agents.evaluation_agent import evaluate_project
@@ -71,6 +71,11 @@ def api_envs() -> dict:
     return get_conda_envs()
 
 
+@app.get("/api/system/readiness")
+def api_runtime_readiness(refresh: bool = False) -> dict:
+    return get_runtime_readiness(force=refresh)
+
+
 @app.get("/api/catalog")
 def api_catalog() -> dict:
     return get_catalog()

backend/app/modules/system/service.py

@@ -1,11 +1,70 @@
 from __future__ import annotations
 
+import json
 import shutil
 import subprocess
+import threading
+import time
+from contextlib import contextmanager
 from pathlib import Path
+from typing import Any
+
+import fcntl
 
 from ...config import settings
 
+READINESS_CACHE_SECONDS = 300
+_readiness_cache: tuple[float, dict[str, Any]] | None = None
+_readiness_thread_lock = threading.Lock()
+
+
+@contextmanager
+def readiness_probe_lock():
+    settings.project_root.joinpath("var").mkdir(parents=True, exist_ok=True)
+    lock_path = settings.project_root / "var" / "runtime_readiness.lock"
+    with _readiness_thread_lock, lock_path.open("w") as lock_file:
+        fcntl.flock(lock_file, fcntl.LOCK_EX)
+        try:
+            yield
+        finally:
+            fcntl.flock(lock_file, fcntl.LOCK_UN)
+
+
+def runtime_environment_specs() -> list[dict[str, Any]]:
+    return [
+        {
+            "role": "backend_task",
+            "name": settings.task_conda_env,
+            "label": "Backend, SegModel, YOLO",
+            "env_file": "envs/seg_smp.yml",
+            "required_imports": [
+                {"module": "fastapi", "package": "fastapi"},
+                {"module": "uvicorn", "package": "uvicorn"},
+                {"module": "torch", "package": "torch"},
+                {"module": "cv2", "package": "opencv-python"},
+                {"module": "segmentation_models_pytorch", "package": "segmentation-models-pytorch"},
+                {"module": "ultralytics", "package": "ultralytics"},
+                {"module": "albumentations", "package": "albumentations"},
+                {"module": "mmengine", "package": "mmengine"},
+                {"module": "mmseg", "package": "mmsegmentation"},
+            ],
+        },
+        {
+            "role": "mmseg_full",
+            "name": settings.mmseg_conda_env,
+            "label": "MMSeg full mmcv runtime",
+            "env_file": "envs/seg_mmcv.yml",
+            "required_imports": [
+                {"module": "torch", "package": "torch"},
+                {"module": "cv2", "package": "opencv-python"},
+                {"module": "mmcv", "package": "mmcv"},
+                {"module": "mmcv._ext", "package": "mmcv"},
+                {"module": "mmengine", "package": "mmengine"},
+                {"module": "mmseg", "package": "mmsegmentation"},
+            ],
+        },
+    ]
+
 
 def parse_nvidia_smi_csv(output: str) -> list[dict]:
     gpus: list[dict] = []
@@ -63,6 +122,152 @@ def get_conda_envs() -> dict:
     return {"available": True, "envs": envs, "task_default": settings.task_conda_env, "mmseg_default": settings.mmseg_conda_env}
 
 
+def probe_code(required_imports: list[dict[str, str]]) -> str:
+    imports_json = json.dumps(required_imports, ensure_ascii=False)
+    return f"""
+import importlib
+import importlib.metadata as metadata
+import json
+import platform
+import sys
+
+required = {imports_json}
+checks = []
+for item in required:
+    module_name = item["module"]
+    package_name = item.get("package") or module_name
+    try:
+        module = importlib.import_module(module_name)
+        version = getattr(module, "__version__", None)
+        if version is None:
+            try:
+                version = metadata.version(package_name)
+            except Exception:
+                version = None
+        checks.append({{
+            "module": module_name,
+            "package": package_name,
+            "passed": True,
+            "version": str(version) if version is not None else None,
+        }})
+    except Exception as exc:
+        checks.append({{
+            "module": module_name,
+            "package": package_name,
+            "passed": False,
+            "error": f"{{type(exc).__name__}}: {{exc}}",
+        }})
+
+extra = {{"python": sys.version.split()[0], "platform": platform.platform()}}
+try:
+    import torch
+    extra["torch_cuda_available"] = bool(torch.cuda.is_available())
+    extra["torch_cuda"] = getattr(torch.version, "cuda", None)
+except Exception:
+    pass
+print(json.dumps({{"checks": checks, "extra": extra}}, ensure_ascii=False))
+"""
+
+
+def parse_probe_stdout(stdout: str) -> dict[str, Any]:
+    for line in reversed(stdout.splitlines()):
+        text = line.strip()
+        if not text or not text.startswith("{"):
+            continue
+        return json.loads(text)
+    raise ValueError("probe did not emit JSON")
+
+
+def inspect_conda_env(env_name: str, required_imports: list[dict[str, str]], timeout: int = 45, retries: int = 1) -> dict[str, Any]:
+    command = ["conda", "run", "-n", env_name, "python", "-c", probe_code(required_imports)]
+    attempts = []
+    result: subprocess.CompletedProcess[str] | None = None
+    for attempt in range(retries + 1):
+        result = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
+        attempts.append({"attempt": attempt + 1, "returncode": result.returncode})
+        if result.returncode == 0 or result.returncode not in {139, -11} or attempt >= retries:
+            break
+        time.sleep(2)
+    if result is None:
+        raise RuntimeError("probe did not run")
+    detail = {
+        "command": command[:4] + ["..."],
+        "returncode": result.returncode,
+        "attempts": attempts,
+        "stdout_tail": result.stdout[-2000:],
+        "stderr_tail": result.stderr[-2000:],
+    }
+    if result.returncode != 0:
+        return {"passed": False, "checks": [], "extra": {}, "detail": detail}
+    try:
+        parsed = parse_probe_stdout(result.stdout)
+    except Exception as exc:
+        return {"passed": False, "checks": [], "extra": {}, "detail": {**detail, "error": str(exc)}}
+    checks = parsed.get("checks", [])
+    return {
+        "passed": bool(checks) and all(item.get("passed") for item in checks),
+        "checks": checks,
+        "extra": parsed.get("extra", {}),
+        "detail": detail,
+    }
+
+
+def get_runtime_readiness(force: bool = False) -> dict[str, Any]:
+    global _readiness_cache
+    now = time.time()
+    if not force and _readiness_cache and now - _readiness_cache[0] < READINESS_CACHE_SECONDS:
+        cached = dict(_readiness_cache[1])
+        cached["cached"] = True
+        return cached
+
+    with readiness_probe_lock():
+        now = time.time()
+        if not force and _readiness_cache and now - _readiness_cache[0] < READINESS_CACHE_SECONDS:
+            cached = dict(_readiness_cache[1])
+            cached["cached"] = True
+            return cached
+
+        conda = get_conda_envs()
+        env_paths = {item["name"]: item["path"] for item in conda.get("envs", [])}
+        envs: list[dict[str, Any]] = []
+        for spec in runtime_environment_specs():
+            env_name = spec["name"]
+            exists = env_name in env_paths
+            env_report: dict[str, Any] = {
+                "role": spec["role"],
+                "name": env_name,
+                "label": spec["label"],
+                "env_file": spec["env_file"],
+                "path": env_paths.get(env_name),
+                "exists": exists,
+                "passed": False,
+                "checks": [],
+                "extra": {},
+            }
+            if exists:
+                probe = inspect_conda_env(env_name, spec["required_imports"])
+                env_report.update(probe)
+            envs.append(env_report)
+
+        payload = {
+            "available": bool(conda.get("available")),
+            "passed": bool(conda.get("available")) and all(item["passed"] for item in envs),
+            "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now)),
+            "cache_seconds": READINESS_CACHE_SECONDS,
+            "cached": False,
+            "envs": envs,
+            "specs": {
+                "bootstrap_script": "scripts/bootstrap_conda_envs.sh",
+                "verify_script": "scripts/verify_runtime_envs.py",
+                "env_files": [spec["env_file"] for spec in runtime_environment_specs()],
+                "task_default": settings.task_conda_env,
+                "mmseg_default": settings.mmseg_conda_env,
+            },
+        }
+        _readiness_cache = (now, payload)
+        return payload
+
+
 def disk_usage() -> dict:
     usage = shutil.disk_usage(settings.source_root)
     return {

backend/tests/test_system_service.py

@@ -1,4 +1,5 @@
-from app.modules.system.service import parse_nvidia_smi_csv
+from app.modules.system import service
+from app.modules.system.service import get_runtime_readiness, parse_nvidia_smi_csv, parse_probe_stdout
 
 
 def test_parse_nvidia_smi_csv():
@@ -16,3 +17,57 @@ def test_parse_nvidia_smi_csv():
         }
     ]
 
+
+def test_parse_probe_stdout_uses_last_json_line():
+    parsed = parse_probe_stdout(
+        "warning before json\n"
+        '{"checks":[{"module":"torch","passed":true,"version":"2.6.0"}],"extra":{"python":"3.11"}}\n'
+    )
+
+    assert parsed["checks"][0]["module"] == "torch"
+    assert parsed["extra"]["python"] == "3.11"
+
+
+def test_runtime_readiness_marks_missing_env(monkeypatch):
+    monkeypatch.setattr(service, "_readiness_cache", None)
+    monkeypatch.setattr(service, "get_conda_envs", lambda: {"available": True, "envs": []})
+
+    readiness = get_runtime_readiness(force=True)
+
+    assert readiness["passed"] is False
+    assert all(not item["exists"] for item in readiness["envs"])
+
+
+def test_runtime_readiness_aggregates_probe_results(monkeypatch):
+    monkeypatch.setattr(service, "_readiness_cache", None)
+    specs = [
+        {"role": "task", "name": "seg_smp", "label": "task", "env_file": "envs/seg_smp.yml", "required_imports": []},
+        {"role": "mmseg", "name": "seg_mmcv", "label": "mmseg", "env_file": "envs/seg_mmcv.yml", "required_imports": []},
+    ]
+    monkeypatch.setattr(service, "runtime_environment_specs", lambda: specs)
+    monkeypatch.setattr(
+        service,
+        "get_conda_envs",
+        lambda: {
+            "available": True,
+            "envs": [
+                {"name": "seg_smp", "path": "/envs/seg_smp"},
+                {"name": "seg_mmcv", "path": "/envs/seg_mmcv"},
+            ],
+        },
+    )
+    monkeypatch.setattr(
+        service,
+        "inspect_conda_env",
+        lambda name, imports: {
+            "passed": True,
+            "checks": [{"module": "torch", "passed": True}],
+            "extra": {"python": "3.11"},
+        },
+    )
+
+    readiness = get_runtime_readiness(force=True)
+
+    assert readiness["passed"] is True
+    assert readiness["envs"][0]["path"] == "/envs/seg_smp"
+    assert readiness["specs"]["env_files"] == ["envs/seg_smp.yml", "envs/seg_mmcv.yml"]