diff --git a/README.md b/README.md index 5158b94..bde3a25 100644 --- a/README.md +++ b/README.md @@ -197,8 +197,11 @@ scripts/check_no_weight_git.sh ``` For a fast non-training validation pass, run agents with -`SEG_VALIDATE_DEEP=0`. The browser dashboard exposes the same readiness, -coverage, GPU, weight, result, and agent checks through the UI. +`PYTHONPATH=backend conda run -n seg_smp python scripts/run_agents.py --no-deep`. +Add `--live`, `--acceptance`, or `--real` only after the backend and frontend +are running and you want HTTP endpoint, smoke, or real-workspace checks. The +browser dashboard exposes the same readiness, coverage, GPU, weight, result, +and agent checks through the UI. The web UI includes a dataset bench for creating upload workspaces, uploading images/labels/masks, and jumping into the existing rename, PNG conversion, @@ -369,6 +372,8 @@ non-training validation pass is needed. The web dashboard calls validation in light mode by default: `/api/agents/validate?run_build=false&run_acceptance=false&run_deep=false`. -Pass `run_acceptance=true`, `run_real=true`, or `run_deep=true` only when you -explicitly want the agent to launch the heavier runtime acceptance checks from -the browser/API. +Pass `run_live=true`, `run_acceptance=true`, `run_real=true`, or +`run_deep=true` only when you explicitly want the agent to launch live endpoint +or heavier runtime acceptance checks from the browser/API. Smoke and real data +acceptance automatically enable the live backend checks because they submit +jobs through the API. diff --git a/backend/app/agents/evaluation_agent.py b/backend/app/agents/evaluation_agent.py index 761f22a..a03c253 100644 --- a/backend/app/agents/evaluation_agent.py +++ b/backend/app/agents/evaluation_agent.py @@ -134,10 +134,19 @@ def evaluate_project() -> dict: if not suggestions: suggestions.append("Current platform covers the requested control-plane features, uploaded YOLO dataset train/predict/heatmap actions, live uploaded-data YOLO predict/heatmap acceptance, real workspace data acceptance, and synthetic deep training acceptance; next focus is a longer operator-run task on a full dataset.") - score = sum(1 for item in checks if item["passed"]) / max(len(checks), 1) + passed_count = sum(1 for item in checks if item["passed"]) + total_count = max(len(checks), 1) + score = passed_count / total_count + passed = passed_count == len(checks) return { "agent": "evaluation_suggestion_agent", + "passed": passed, "score": round(score, 3), + "summary": { + "passed_checks": passed_count, + "total_checks": len(checks), + "missing_checks": [item["name"] for item in checks if not item["passed"]], + }, "checks": checks, "suggestions": suggestions, } diff --git a/backend/app/agents/validation_agent.py b/backend/app/agents/validation_agent.py index 6be8be9..43b5542 100644 --- a/backend/app/agents/validation_agent.py +++ b/backend/app/agents/validation_agent.py @@ -47,6 +47,7 @@ def validate_project( run_acceptance: bool | None = None, run_deep: bool | None = None, run_real: bool | None = None, + run_live: bool | None = None, ) -> dict: """Validate current runtime readiness without launching heavy training.""" checks = [] @@ -67,7 +68,14 @@ def validate_project( checks.append({"name": "training_curves_detected", "passed": len(curves) >= 1, "detail": {"count": len(curves), "examples": [item["relative_path"] for item in curves[:5]]}}) parsed_progress = parse_progress("Epoch(train) [2][25/50] lr: 1e-3\n", status="running") checks.append({"name": "job_progress_parser", "passed": parsed_progress["stage"] == "training" and parsed_progress["percent"] == 50, "detail": parsed_progress}) - checks.append({"name": "gpus_query", "passed": bool(get_gpus().get("available"))}) + gpus = get_gpus() + checks.append( + { + "name": "gpus_query", + "passed": "available" in gpus and isinstance(gpus.get("gpus"), list), + "detail": gpus, + } + ) env_names = [item["name"] for item in get_conda_envs().get("envs", [])] checks.append({"name": "task_env_exists", "passed": settings.task_conda_env in env_names, "detail": {"env": settings.task_conda_env}}) checks.append({"name": "mmseg_env_exists", "passed": settings.mmseg_conda_env in env_names, "detail": {"env": settings.mmseg_conda_env}}) @@ -111,7 +119,13 @@ def validate_project( no_weight = _run(["bash", "scripts/check_no_weight_git.sh"], cwd=settings.project_root) checks.append({"name": "no_weight_in_git", "passed": no_weight["passed"], "detail": no_weight}) - if os.getenv("SEG_VALIDATE_LIVE", "1") == "1": + acceptance_enabled = run_acceptance if run_acceptance is not None else os.getenv("SEG_VALIDATE_ACCEPTANCE", "0") == "1" + deep_enabled = run_deep if run_deep is not None else os.getenv("SEG_VALIDATE_DEEP", "1") == "1" + real_enabled = run_real if run_real is not None else os.getenv("SEG_VALIDATE_REAL", "0") == "1" + live_enabled = run_live if run_live is not None else os.getenv("SEG_VALIDATE_LIVE", "0") == "1" + live_enabled = live_enabled or acceptance_enabled or real_enabled + + if live_enabled: backend_url = os.getenv("SEG_VALIDATE_BACKEND_URL", "http://127.0.0.1:8010") frontend_url = os.getenv("SEG_VALIDATE_FRONTEND_URL", "http://127.0.0.1:5173") health = _fetch(f"{backend_url}/api/health") @@ -146,9 +160,6 @@ def validate_project( checks.append({"name": "live_coverage_api", "passed": live_coverage["passed"] and '"task_build_passed":true' in live_coverage.get("body", "").replace(" ", ""), "detail": live_coverage}) checks.append({"name": "live_training_curves_api", "passed": live_curves["passed"] and live_curves.get("body", "").lstrip().startswith("["), "detail": live_curves}) checks.append({"name": "live_frontend_index", "passed": frontend["passed"] and "Seg Data Server" in frontend.get("body", ""), "detail": frontend}) - acceptance_enabled = run_acceptance if run_acceptance is not None else os.getenv("SEG_VALIDATE_ACCEPTANCE", "1") == "1" - deep_enabled = run_deep if run_deep is not None else os.getenv("SEG_VALIDATE_DEEP", "1") == "1" - real_enabled = run_real if run_real is not None else os.getenv("SEG_VALIDATE_REAL", "0") == "1" if acceptance_enabled: acceptance = run_live_acceptance(backend_url) checks.append({"name": "live_acceptance_smoke", "passed": acceptance["passed"], "detail": acceptance}) @@ -158,6 +169,9 @@ def validate_project( if deep_enabled: deep_acceptance = run_deep_acceptance() checks.append({"name": "deep_training_acceptance", "passed": deep_acceptance["passed"], "detail": deep_acceptance}) + elif deep_enabled: + deep_acceptance = run_deep_acceptance() + checks.append({"name": "deep_training_acceptance", "passed": deep_acceptance["passed"], "detail": deep_acceptance}) if run_build: tests = _run(["conda", "run", "-n", settings.backend_conda_env, "python", "-m", "pytest", "-q"], cwd=settings.project_root, timeout=120) diff --git a/backend/app/capabilities.py b/backend/app/capabilities.py index ecb3148..1a68705 100644 --- a/backend/app/capabilities.py +++ b/backend/app/capabilities.py @@ -3,7 +3,7 @@ from __future__ import annotations import time from typing import Any -from .acceptance import latest_acceptance_report, latest_deep_acceptance_report +from .acceptance import latest_acceptance_report, latest_deep_acceptance_report, latest_real_acceptance_report from .catalog import get_catalog from .coverage import get_coverage_report from .modules.dataset.service import list_uploaded_datasets @@ -156,6 +156,7 @@ def get_capability_matrix() -> dict[str, Any]: manifest = load_manifest() gpus = get_gpus() acceptance = latest_acceptance_report() + real_acceptance = latest_real_acceptance_report() deep_acceptance = latest_deep_acceptance_report() all_tasks = catalog["task_types"] @@ -270,6 +271,12 @@ def get_capability_matrix() -> dict[str, Any]: "passed": bool(deep_acceptance.get("passed")), "detail": deep_acceptance.get("run_id", "not run"), }, + { + "id": "real_workspace_acceptance", + "label": "真实数据验收", + "passed": bool(real_acceptance.get("passed")), + "detail": real_acceptance.get("run_id", "not run"), + }, { "id": "weights_manifest", "label": "权重清单", @@ -294,6 +301,7 @@ def get_capability_matrix() -> dict[str, Any]: "weights": manifest.get("count", 0), "gpus_available": bool(gpus.get("available")), "acceptance_passed": bool(acceptance.get("passed")), + "real_acceptance_passed": bool(real_acceptance.get("passed")), "deep_acceptance_passed": bool(deep_acceptance.get("passed")), }, "requirements": requirements, diff --git a/backend/app/main.py b/backend/app/main.py index e16c575..3bbb6b7 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -290,5 +290,17 @@ def api_agent_evaluate() -> dict: @app.get("/api/agents/validate") -def api_agent_validate(run_build: bool = False, run_acceptance: bool = False, run_deep: bool = False, run_real: bool = False) -> dict: - return validate_project(run_build=run_build, run_acceptance=run_acceptance, run_deep=run_deep, run_real=run_real) +def api_agent_validate( + run_build: bool = False, + run_acceptance: bool = False, + run_deep: bool = False, + run_real: bool = False, + run_live: bool | None = None, +) -> dict: + return validate_project( + run_build=run_build, + run_acceptance=run_acceptance, + run_deep=run_deep, + run_real=run_real, + run_live=run_live, + ) diff --git a/backend/tests/test_agents.py b/backend/tests/test_agents.py index 82d5fc1..d8818ed 100644 --- a/backend/tests/test_agents.py +++ b/backend/tests/test_agents.py @@ -5,13 +5,16 @@ from app.agents.validation_agent import validate_project def test_evaluation_agent_returns_checks(): result = evaluate_project() assert result["agent"] == "evaluation_suggestion_agent" + assert result["passed"] is True assert result["checks"] + assert result["summary"]["passed_checks"] == result["summary"]["total_checks"] checks = {item["name"]: item["passed"] for item in result["checks"]} assert checks["real_workspace_acceptance"] is True def test_validation_agent_lightweight(monkeypatch): monkeypatch.setenv("SEG_VALIDATE_LIVE", "0") - result = validate_project(run_build=False) + result = validate_project(run_build=False, run_deep=False) assert result["agent"] == "validation_agent" + assert result["passed"] is True assert any(item["name"] == "catalog_has_yolo_heatmap" for item in result["checks"]) diff --git a/backend/tests/test_capabilities.py b/backend/tests/test_capabilities.py index 425d86c..88bdc96 100644 --- a/backend/tests/test_capabilities.py +++ b/backend/tests/test_capabilities.py @@ -19,3 +19,4 @@ def test_capability_matrix_tracks_user_requirements(): assert requirements["runtime_readiness"]["passed"] is True assert requirements["yolo_heatmap"]["passed"] is True assert requirements["training_curves"]["passed"] is True + assert requirements["real_workspace_acceptance"]["passed"] is True diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx index 7ebef81..fc07718 100644 --- a/frontend/src/main.tsx +++ b/frontend/src/main.tsx @@ -287,7 +287,13 @@ type AgentCheck = { type EvaluationAgentPayload = { agent: string; + passed: boolean; score: number; + summary?: { + passed_checks: number; + total_checks: number; + missing_checks: string[]; + }; checks: AgentCheck[]; suggestions: string[]; }; @@ -1203,11 +1209,11 @@ function App() {

Evaluation Agent

评价建议

- = 1 ? "success" : "queued"} /> +
{Math.round((agentEvaluation?.score ?? 0) * 100)}% - {agentEvaluation?.checks.filter((item) => item.passed).length ?? 0}/{agentEvaluation?.checks.length ?? 0} checks passed + {agentEvaluation?.summary ? `${agentEvaluation.summary.passed_checks}/${agentEvaluation.summary.total_checks}` : `${agentEvaluation?.checks.filter((item) => item.passed).length ?? 0}/${agentEvaluation?.checks.length ?? 0}`} checks passed
{(agentEvaluation?.suggestions ?? ["等待评价 agent 返回建议。"]).slice(0, 6).map((item, index) => ( diff --git a/scripts/run_agents.py b/scripts/run_agents.py index 882dfa4..0286e2d 100644 --- a/scripts/run_agents.py +++ b/scripts/run_agents.py @@ -16,20 +16,29 @@ from app.agents.validation_agent import validate_project # noqa: E402 def main() -> None: parser = argparse.ArgumentParser(description="Run local evaluation and validation agents.") parser.add_argument("--build", action="store_true", help="also run pytest and frontend build") + parser.add_argument("--live", action="store_true", help="also check live backend/frontend HTTP endpoints") + parser.add_argument("--acceptance", action="store_true", help="run the lightweight live acceptance smoke") + parser.add_argument("--real", action="store_true", help="run real workspace data acceptance through the live backend") + parser.add_argument("--no-deep", action="store_true", help="skip synthetic deep training acceptance") parser.add_argument("--out", default="var/agent_reports/latest.json") args = parser.parse_args() report = { "evaluation": evaluate_project(), - "validation": validate_project(run_build=args.build), + "validation": validate_project( + run_build=args.build, + run_live=args.live, + run_acceptance=args.acceptance, + run_real=args.real, + run_deep=not args.no_deep, + ), } out = ROOT / args.out out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") print(json.dumps(report, ensure_ascii=False, indent=2)) - if not report["validation"]["passed"]: + if not report["evaluation"]["passed"] or not report["validation"]["passed"]: raise SystemExit(1) if __name__ == "__main__": main() -