Tighten agent validation gates

README.md

@@ -197,8 +197,11 @@ scripts/check_no_weight_git.sh
 ```
 
 For a fast non-training validation pass, run agents with
-`SEG_VALIDATE_DEEP=0`. The browser dashboard exposes the same readiness,
+`PYTHONPATH=backend conda run -n seg_smp python scripts/run_agents.py --no-deep`.
-coverage, GPU, weight, result, and agent checks through the UI.
+Add `--live`, `--acceptance`, or `--real` only after the backend and frontend
+are running and you want HTTP endpoint, smoke, or real-workspace checks. The
+browser dashboard exposes the same readiness, coverage, GPU, weight, result,
+and agent checks through the UI.
 
 The web UI includes a dataset bench for creating upload workspaces, uploading
 images/labels/masks, and jumping into the existing rename, PNG conversion,
@@ -369,6 +372,8 @@ non-training validation pass is needed.
 
 The web dashboard calls validation in light mode by default:
 `/api/agents/validate?run_build=false&run_acceptance=false&run_deep=false`.
-Pass `run_acceptance=true`, `run_real=true`, or `run_deep=true` only when you
+Pass `run_live=true`, `run_acceptance=true`, `run_real=true`, or
-explicitly want the agent to launch the heavier runtime acceptance checks from
+`run_deep=true` only when you explicitly want the agent to launch live endpoint
-the browser/API.
+or heavier runtime acceptance checks from the browser/API. Smoke and real data
+acceptance automatically enable the live backend checks because they submit
+jobs through the API.

backend/app/agents/evaluation_agent.py

@@ -134,10 +134,19 @@ def evaluate_project() -> dict:
     if not suggestions:
         suggestions.append("Current platform covers the requested control-plane features, uploaded YOLO dataset train/predict/heatmap actions, live uploaded-data YOLO predict/heatmap acceptance, real workspace data acceptance, and synthetic deep training acceptance; next focus is a longer operator-run task on a full dataset.")
 
-    score = sum(1 for item in checks if item["passed"]) / max(len(checks), 1)
+    passed_count = sum(1 for item in checks if item["passed"])
+    total_count = max(len(checks), 1)
+    score = passed_count / total_count
+    passed = passed_count == len(checks)
     return {
         "agent": "evaluation_suggestion_agent",
+        "passed": passed,
         "score": round(score, 3),
+        "summary": {
+            "passed_checks": passed_count,
+            "total_checks": len(checks),
+            "missing_checks": [item["name"] for item in checks if not item["passed"]],
+        },
         "checks": checks,
         "suggestions": suggestions,
     }

backend/app/agents/validation_agent.py

@@ -47,6 +47,7 @@ def validate_project(
     run_acceptance: bool | None = None,
     run_deep: bool | None = None,
     run_real: bool | None = None,
+    run_live: bool | None = None,
 ) -> dict:
     """Validate current runtime readiness without launching heavy training."""
     checks = []
@@ -67,7 +68,14 @@ def validate_project(
     checks.append({"name": "training_curves_detected", "passed": len(curves) >= 1, "detail": {"count": len(curves), "examples": [item["relative_path"] for item in curves[:5]]}})
     parsed_progress = parse_progress("Epoch(train) [2][25/50] lr: 1e-3\n", status="running")
     checks.append({"name": "job_progress_parser", "passed": parsed_progress["stage"] == "training" and parsed_progress["percent"] == 50, "detail": parsed_progress})
-    checks.append({"name": "gpus_query", "passed": bool(get_gpus().get("available"))})
+    gpus = get_gpus()
+    checks.append(
+        {
+            "name": "gpus_query",
+            "passed": "available" in gpus and isinstance(gpus.get("gpus"), list),
+            "detail": gpus,
+        }
+    )
     env_names = [item["name"] for item in get_conda_envs().get("envs", [])]
     checks.append({"name": "task_env_exists", "passed": settings.task_conda_env in env_names, "detail": {"env": settings.task_conda_env}})
     checks.append({"name": "mmseg_env_exists", "passed": settings.mmseg_conda_env in env_names, "detail": {"env": settings.mmseg_conda_env}})
@@ -111,7 +119,13 @@ def validate_project(
     no_weight = _run(["bash", "scripts/check_no_weight_git.sh"], cwd=settings.project_root)
     checks.append({"name": "no_weight_in_git", "passed": no_weight["passed"], "detail": no_weight})
 
-    if os.getenv("SEG_VALIDATE_LIVE", "1") == "1":
+    acceptance_enabled = run_acceptance if run_acceptance is not None else os.getenv("SEG_VALIDATE_ACCEPTANCE", "0") == "1"
+    deep_enabled = run_deep if run_deep is not None else os.getenv("SEG_VALIDATE_DEEP", "1") == "1"
+    real_enabled = run_real if run_real is not None else os.getenv("SEG_VALIDATE_REAL", "0") == "1"
+    live_enabled = run_live if run_live is not None else os.getenv("SEG_VALIDATE_LIVE", "0") == "1"
+    live_enabled = live_enabled or acceptance_enabled or real_enabled
+
+    if live_enabled:
         backend_url = os.getenv("SEG_VALIDATE_BACKEND_URL", "http://127.0.0.1:8010")
         frontend_url = os.getenv("SEG_VALIDATE_FRONTEND_URL", "http://127.0.0.1:5173")
         health = _fetch(f"{backend_url}/api/health")
@@ -146,9 +160,6 @@ def validate_project(
         checks.append({"name": "live_coverage_api", "passed": live_coverage["passed"] and '"task_build_passed":true' in live_coverage.get("body", "").replace(" ", ""), "detail": live_coverage})
         checks.append({"name": "live_training_curves_api", "passed": live_curves["passed"] and live_curves.get("body", "").lstrip().startswith("["), "detail": live_curves})
         checks.append({"name": "live_frontend_index", "passed": frontend["passed"] and "Seg Data Server" in frontend.get("body", ""), "detail": frontend})
-        acceptance_enabled = run_acceptance if run_acceptance is not None else os.getenv("SEG_VALIDATE_ACCEPTANCE", "1") == "1"
-        deep_enabled = run_deep if run_deep is not None else os.getenv("SEG_VALIDATE_DEEP", "1") == "1"
-        real_enabled = run_real if run_real is not None else os.getenv("SEG_VALIDATE_REAL", "0") == "1"
         if acceptance_enabled:
             acceptance = run_live_acceptance(backend_url)
             checks.append({"name": "live_acceptance_smoke", "passed": acceptance["passed"], "detail": acceptance})
@@ -158,6 +169,9 @@ def validate_project(
         if deep_enabled:
             deep_acceptance = run_deep_acceptance()
             checks.append({"name": "deep_training_acceptance", "passed": deep_acceptance["passed"], "detail": deep_acceptance})
+    elif deep_enabled:
+        deep_acceptance = run_deep_acceptance()
+        checks.append({"name": "deep_training_acceptance", "passed": deep_acceptance["passed"], "detail": deep_acceptance})
 
     if run_build:
         tests = _run(["conda", "run", "-n", settings.backend_conda_env, "python", "-m", "pytest", "-q"], cwd=settings.project_root, timeout=120)

backend/app/capabilities.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import time
 from typing import Any
 
-from .acceptance import latest_acceptance_report, latest_deep_acceptance_report
+from .acceptance import latest_acceptance_report, latest_deep_acceptance_report, latest_real_acceptance_report
 from .catalog import get_catalog
 from .coverage import get_coverage_report
 from .modules.dataset.service import list_uploaded_datasets
@@ -156,6 +156,7 @@ def get_capability_matrix() -> dict[str, Any]:
     manifest = load_manifest()
     gpus = get_gpus()
     acceptance = latest_acceptance_report()
+    real_acceptance = latest_real_acceptance_report()
     deep_acceptance = latest_deep_acceptance_report()
 
     all_tasks = catalog["task_types"]
@@ -270,6 +271,12 @@ def get_capability_matrix() -> dict[str, Any]:
             "passed": bool(deep_acceptance.get("passed")),
             "detail": deep_acceptance.get("run_id", "not run"),
         },
+        {
+            "id": "real_workspace_acceptance",
+            "label": "真实数据验收",
+            "passed": bool(real_acceptance.get("passed")),
+            "detail": real_acceptance.get("run_id", "not run"),
+        },
         {
             "id": "weights_manifest",
             "label": "权重清单",
@@ -294,6 +301,7 @@ def get_capability_matrix() -> dict[str, Any]:
             "weights": manifest.get("count", 0),
             "gpus_available": bool(gpus.get("available")),
             "acceptance_passed": bool(acceptance.get("passed")),
+            "real_acceptance_passed": bool(real_acceptance.get("passed")),
             "deep_acceptance_passed": bool(deep_acceptance.get("passed")),
         },
         "requirements": requirements,

backend/app/main.py

@@ -290,5 +290,17 @@ def api_agent_evaluate() -> dict:
 
 
 @app.get("/api/agents/validate")
-def api_agent_validate(run_build: bool = False, run_acceptance: bool = False, run_deep: bool = False, run_real: bool = False) -> dict:
+def api_agent_validate(
-    return validate_project(run_build=run_build, run_acceptance=run_acceptance, run_deep=run_deep, run_real=run_real)
+    run_build: bool = False,
+    run_acceptance: bool = False,
+    run_deep: bool = False,
+    run_real: bool = False,
+    run_live: bool | None = None,
+) -> dict:
+    return validate_project(
+        run_build=run_build,
+        run_acceptance=run_acceptance,
+        run_deep=run_deep,
+        run_real=run_real,
+        run_live=run_live,
+    )

backend/tests/test_agents.py

@@ -5,13 +5,16 @@ from app.agents.validation_agent import validate_project
 def test_evaluation_agent_returns_checks():
     result = evaluate_project()
     assert result["agent"] == "evaluation_suggestion_agent"
+    assert result["passed"] is True
     assert result["checks"]
+    assert result["summary"]["passed_checks"] == result["summary"]["total_checks"]
     checks = {item["name"]: item["passed"] for item in result["checks"]}
     assert checks["real_workspace_acceptance"] is True
 
 
 def test_validation_agent_lightweight(monkeypatch):
     monkeypatch.setenv("SEG_VALIDATE_LIVE", "0")
-    result = validate_project(run_build=False)
+    result = validate_project(run_build=False, run_deep=False)
     assert result["agent"] == "validation_agent"
+    assert result["passed"] is True
     assert any(item["name"] == "catalog_has_yolo_heatmap" for item in result["checks"])

backend/tests/test_capabilities.py

@@ -19,3 +19,4 @@ def test_capability_matrix_tracks_user_requirements():
     assert requirements["runtime_readiness"]["passed"] is True
     assert requirements["yolo_heatmap"]["passed"] is True
     assert requirements["training_curves"]["passed"] is True
+    assert requirements["real_workspace_acceptance"]["passed"] is True

frontend/src/main.tsx

@@ -287,7 +287,13 @@ type AgentCheck = {
 
 type EvaluationAgentPayload = {
   agent: string;
+  passed: boolean;
   score: number;
+  summary?: {
+    passed_checks: number;
+    total_checks: number;
+    missing_checks: string[];
+  };
   checks: AgentCheck[];
   suggestions: string[];
 };
@@ -1203,11 +1209,11 @@ function App() {
                 <p className="eyebrow">Evaluation Agent</p>
                 <h2>评价建议</h2>
               </div>
-              <StatusPill status={(agentEvaluation?.score ?? 0) >= 1 ? "success" : "queued"} />
+              <StatusPill status={agentEvaluation?.passed ? "success" : "queued"} />
             </div>
             <div className="agentScore">
               <strong>{Math.round((agentEvaluation?.score ?? 0) * 100)}%</strong>
-              <span>{agentEvaluation?.checks.filter((item) => item.passed).length ?? 0}/{agentEvaluation?.checks.length ?? 0} checks passed</span>
+              <span>{agentEvaluation?.summary ? `${agentEvaluation.summary.passed_checks}/${agentEvaluation.summary.total_checks}` : `${agentEvaluation?.checks.filter((item) => item.passed).length ?? 0}/${agentEvaluation?.checks.length ?? 0}`} checks passed</span>
             </div>
             <div className="suggestionList">
               {(agentEvaluation?.suggestions ?? ["等待评价 agent 返回建议。"]).slice(0, 6).map((item, index) => (

scripts/run_agents.py

@@ -16,20 +16,29 @@ from app.agents.validation_agent import validate_project  # noqa: E402
 def main() -> None:
     parser = argparse.ArgumentParser(description="Run local evaluation and validation agents.")
     parser.add_argument("--build", action="store_true", help="also run pytest and frontend build")
+    parser.add_argument("--live", action="store_true", help="also check live backend/frontend HTTP endpoints")
+    parser.add_argument("--acceptance", action="store_true", help="run the lightweight live acceptance smoke")
+    parser.add_argument("--real", action="store_true", help="run real workspace data acceptance through the live backend")
+    parser.add_argument("--no-deep", action="store_true", help="skip synthetic deep training acceptance")
     parser.add_argument("--out", default="var/agent_reports/latest.json")
     args = parser.parse_args()
     report = {
         "evaluation": evaluate_project(),
-        "validation": validate_project(run_build=args.build),
+        "validation": validate_project(
+            run_build=args.build,
+            run_live=args.live,
+            run_acceptance=args.acceptance,
+            run_real=args.real,
+            run_deep=not args.no_deep,
+        ),
     }
     out = ROOT / args.out
     out.parent.mkdir(parents=True, exist_ok=True)
     out.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
     print(json.dumps(report, ensure_ascii=False, indent=2))
-    if not report["validation"]["passed"]:
+    if not report["evaluation"]["passed"] or not report["validation"]["passed"]:
         raise SystemExit(1)
 
 
 if __name__ == "__main__":
     main()
-