Tighten agent validation gates

2026-06-30 23:24:31 +08:00
parent 53b81dd04d
commit 5055084788
9 changed files with 87 additions and 20 deletions
--- a/backend/app/agents/evaluation_agent.py
+++ b/backend/app/agents/evaluation_agent.py
@@ -134,10 +134,19 @@ def evaluate_project() -> dict:
    if not suggestions:
        suggestions.append("Current platform covers the requested control-plane features, uploaded YOLO dataset train/predict/heatmap actions, live uploaded-data YOLO predict/heatmap acceptance, real workspace data acceptance, and synthetic deep training acceptance; next focus is a longer operator-run task on a full dataset.")

-    score = sum(1 for item in checks if item["passed"]) / max(len(checks), 1)
+    passed_count = sum(1 for item in checks if item["passed"])
+    total_count = max(len(checks), 1)
+    score = passed_count / total_count
+    passed = passed_count == len(checks)
    return {
        "agent": "evaluation_suggestion_agent",
+        "passed": passed,
        "score": round(score, 3),
+        "summary": {
+            "passed_checks": passed_count,
+            "total_checks": len(checks),
+            "missing_checks": [item["name"] for item in checks if not item["passed"]],
+        },
        "checks": checks,
        "suggestions": suggestions,
    }
--- a/backend/app/agents/validation_agent.py
+++ b/backend/app/agents/validation_agent.py
@@ -47,6 +47,7 @@ def validate_project(
    run_acceptance: bool | None = None,
    run_deep: bool | None = None,
    run_real: bool | None = None,
+    run_live: bool | None = None,
 ) -> dict:
    """Validate current runtime readiness without launching heavy training."""
    checks = []
@@ -67,7 +68,14 @@ def validate_project(
    checks.append({"name": "training_curves_detected", "passed": len(curves) >= 1, "detail": {"count": len(curves), "examples": [item["relative_path"] for item in curves[:5]]}})
    parsed_progress = parse_progress("Epoch(train) [2][25/50] lr: 1e-3\n", status="running")
    checks.append({"name": "job_progress_parser", "passed": parsed_progress["stage"] == "training" and parsed_progress["percent"] == 50, "detail": parsed_progress})
-    checks.append({"name": "gpus_query", "passed": bool(get_gpus().get("available"))})
+    gpus = get_gpus()
+    checks.append(
+        {
+            "name": "gpus_query",
+            "passed": "available" in gpus and isinstance(gpus.get("gpus"), list),
+            "detail": gpus,
+        }
+    )
    env_names = [item["name"] for item in get_conda_envs().get("envs", [])]
    checks.append({"name": "task_env_exists", "passed": settings.task_conda_env in env_names, "detail": {"env": settings.task_conda_env}})
    checks.append({"name": "mmseg_env_exists", "passed": settings.mmseg_conda_env in env_names, "detail": {"env": settings.mmseg_conda_env}})
@@ -111,7 +119,13 @@ def validate_project(
    no_weight = _run(["bash", "scripts/check_no_weight_git.sh"], cwd=settings.project_root)
    checks.append({"name": "no_weight_in_git", "passed": no_weight["passed"], "detail": no_weight})

-    if os.getenv("SEG_VALIDATE_LIVE", "1") == "1":
+    acceptance_enabled = run_acceptance if run_acceptance is not None else os.getenv("SEG_VALIDATE_ACCEPTANCE", "0") == "1"
+    deep_enabled = run_deep if run_deep is not None else os.getenv("SEG_VALIDATE_DEEP", "1") == "1"
+    real_enabled = run_real if run_real is not None else os.getenv("SEG_VALIDATE_REAL", "0") == "1"
+    live_enabled = run_live if run_live is not None else os.getenv("SEG_VALIDATE_LIVE", "0") == "1"
+    live_enabled = live_enabled or acceptance_enabled or real_enabled
+
+    if live_enabled:
        backend_url = os.getenv("SEG_VALIDATE_BACKEND_URL", "http://127.0.0.1:8010")
        frontend_url = os.getenv("SEG_VALIDATE_FRONTEND_URL", "http://127.0.0.1:5173")
        health = _fetch(f"{backend_url}/api/health")
@@ -146,9 +160,6 @@ def validate_project(
        checks.append({"name": "live_coverage_api", "passed": live_coverage["passed"] and '"task_build_passed":true' in live_coverage.get("body", "").replace(" ", ""), "detail": live_coverage})
        checks.append({"name": "live_training_curves_api", "passed": live_curves["passed"] and live_curves.get("body", "").lstrip().startswith("["), "detail": live_curves})
        checks.append({"name": "live_frontend_index", "passed": frontend["passed"] and "Seg Data Server" in frontend.get("body", ""), "detail": frontend})
-        acceptance_enabled = run_acceptance if run_acceptance is not None else os.getenv("SEG_VALIDATE_ACCEPTANCE", "1") == "1"
-        deep_enabled = run_deep if run_deep is not None else os.getenv("SEG_VALIDATE_DEEP", "1") == "1"
-        real_enabled = run_real if run_real is not None else os.getenv("SEG_VALIDATE_REAL", "0") == "1"
        if acceptance_enabled:
            acceptance = run_live_acceptance(backend_url)
            checks.append({"name": "live_acceptance_smoke", "passed": acceptance["passed"], "detail": acceptance})
@@ -158,6 +169,9 @@ def validate_project(
        if deep_enabled:
            deep_acceptance = run_deep_acceptance()
            checks.append({"name": "deep_training_acceptance", "passed": deep_acceptance["passed"], "detail": deep_acceptance})
+    elif deep_enabled:
+        deep_acceptance = run_deep_acceptance()
+        checks.append({"name": "deep_training_acceptance", "passed": deep_acceptance["passed"], "detail": deep_acceptance})

    if run_build:
        tests = _run(["conda", "run", "-n", settings.backend_conda_env, "python", "-m", "pytest", "-q"], cwd=settings.project_root, timeout=120)
--- a/backend/app/capabilities.py
+++ b/backend/app/capabilities.py
@@ -3,7 +3,7 @@ from __future__ import annotations
 import time
 from typing import Any

-from .acceptance import latest_acceptance_report, latest_deep_acceptance_report
+from .acceptance import latest_acceptance_report, latest_deep_acceptance_report, latest_real_acceptance_report
 from .catalog import get_catalog
 from .coverage import get_coverage_report
 from .modules.dataset.service import list_uploaded_datasets
@@ -156,6 +156,7 @@ def get_capability_matrix() -> dict[str, Any]:
    manifest = load_manifest()
    gpus = get_gpus()
    acceptance = latest_acceptance_report()
+    real_acceptance = latest_real_acceptance_report()
    deep_acceptance = latest_deep_acceptance_report()

    all_tasks = catalog["task_types"]
@@ -270,6 +271,12 @@ def get_capability_matrix() -> dict[str, Any]:
            "passed": bool(deep_acceptance.get("passed")),
            "detail": deep_acceptance.get("run_id", "not run"),
        },
+        {
+            "id": "real_workspace_acceptance",
+            "label": "真实数据验收",
+            "passed": bool(real_acceptance.get("passed")),
+            "detail": real_acceptance.get("run_id", "not run"),
+        },
        {
            "id": "weights_manifest",
            "label": "权重清单",
@@ -294,6 +301,7 @@ def get_capability_matrix() -> dict[str, Any]:
            "weights": manifest.get("count", 0),
            "gpus_available": bool(gpus.get("available")),
            "acceptance_passed": bool(acceptance.get("passed")),
+            "real_acceptance_passed": bool(real_acceptance.get("passed")),
            "deep_acceptance_passed": bool(deep_acceptance.get("passed")),
        },
        "requirements": requirements,
--- a/backend/app/main.py
+++ b/backend/app/main.py
@@ -290,5 +290,17 @@ def api_agent_evaluate() -> dict:


@app.get("/api/agents/validate")
-def api_agent_validate(run_build: bool = False, run_acceptance: bool = False, run_deep: bool = False, run_real: bool = False) -> dict:
-    return validate_project(run_build=run_build, run_acceptance=run_acceptance, run_deep=run_deep, run_real=run_real)
+def api_agent_validate(
+    run_build: bool = False,
+    run_acceptance: bool = False,
+    run_deep: bool = False,
+    run_real: bool = False,
+    run_live: bool | None = None,
+) -> dict:
+    return validate_project(
+        run_build=run_build,
+        run_acceptance=run_acceptance,
+        run_deep=run_deep,
+        run_real=run_real,
+        run_live=run_live,
+    )