admin/Seg_Data_Server_Net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tighten agent validation gates

Browse Source
This commit is contained in:
admin
2026-06-30 23:24:31 +08:00
parent 53b81dd04d
commit 5055084788
9 changed files with 87 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
README.md
View File

@@ -197,8 +197,11 @@ scripts/check_no_weight_git.sh
```
For a fast non-training validation pass, run agents with
`SEG_VALIDATE_DEEP=0`. The browser dashboard exposes the same readiness,
coverage, GPU, weight, result, and agent checks through the UI.
`PYTHONPATH=backend conda run -n seg_smp python scripts/run_agents.py --no-deep`.
Add `--live`, `--acceptance`, or `--real` only after the backend and frontend
are running and you want HTTP endpoint, smoke, or real-workspace checks. The
browser dashboard exposes the same readiness, coverage, GPU, weight, result,
and agent checks through the UI.
The web UI includes a dataset bench for creating upload workspaces, uploading
images/labels/masks, and jumping into the existing rename, PNG conversion,
@@ -369,6 +372,8 @@ non-training validation pass is needed.
The web dashboard calls validation in light mode by default:
`/api/agents/validate?run_build=false&run_acceptance=false&run_deep=false`.
Pass `run_acceptance=true`, `run_real=true`, or `run_deep=true` only when you
explicitly want the agent to launch the heavier runtime acceptance checks from
the browser/API.
Pass `run_live=true`, `run_acceptance=true`, `run_real=true`, or
`run_deep=true` only when you explicitly want the agent to launch live endpoint
or heavier runtime acceptance checks from the browser/API. Smoke and real data
acceptance automatically enable the live backend checks because they submit
jobs through the API.

11
backend/app/agents/evaluation_agent.py
View File

@@ -134,10 +134,19 @@ def evaluate_project() -> dict:
if not suggestions:
suggestions.append("Current platform covers the requested control-plane features, uploaded YOLO dataset train/predict/heatmap actions, live uploaded-data YOLO predict/heatmap acceptance, real workspace data acceptance, and synthetic deep training acceptance; next focus is a longer operator-run task on a full dataset.")
score = sum(1 for item in checks if item["passed"]) / max(len(checks), 1)
passed_count = sum(1 for item in checks if item["passed"])
total_count = max(len(checks), 1)
score = passed_count / total_count
passed = passed_count == len(checks)
return {
"agent": "evaluation_suggestion_agent",
"passed": passed,
"score": round(score, 3),
"summary": {
"passed_checks": passed_count,
"total_checks": len(checks),
"missing_checks": [item["name"] for item in checks if not item["passed"]],
},
"checks": checks,
"suggestions": suggestions,
}

24
backend/app/agents/validation_agent.py
View File

@@ -47,6 +47,7 @@ def validate_project(
run_acceptance: bool | None = None,
run_deep: bool | None = None,
run_real: bool | None = None,
run_live: bool | None = None,
) -> dict:
"""Validate current runtime readiness without launching heavy training."""
checks = []
@@ -67,7 +68,14 @@ def validate_project(
checks.append({"name": "training_curves_detected", "passed": len(curves) >= 1, "detail": {"count": len(curves), "examples": [item["relative_path"] for item in curves[:5]]}})
parsed_progress = parse_progress("Epoch(train) [2][25/50] lr: 1e-3\n", status="running")
checks.append({"name": "job_progress_parser", "passed": parsed_progress["stage"] == "training" and parsed_progress["percent"] == 50, "detail": parsed_progress})
checks.append({"name": "gpus_query", "passed": bool(get_gpus().get("available"))})
gpus = get_gpus()
checks.append(
{
"name": "gpus_query",
"passed": "available" in gpus and isinstance(gpus.get("gpus"), list),
"detail": gpus,
}
)
env_names = [item["name"] for item in get_conda_envs().get("envs", [])]
checks.append({"name": "task_env_exists", "passed": settings.task_conda_env in env_names, "detail": {"env": settings.task_conda_env}})
checks.append({"name": "mmseg_env_exists", "passed": settings.mmseg_conda_env in env_names, "detail": {"env": settings.mmseg_conda_env}})
@@ -111,7 +119,13 @@ def validate_project(
no_weight = _run(["bash", "scripts/check_no_weight_git.sh"], cwd=settings.project_root)
checks.append({"name": "no_weight_in_git", "passed": no_weight["passed"], "detail": no_weight})
if os.getenv("SEG_VALIDATE_LIVE", "1") == "1":
acceptance_enabled = run_acceptance if run_acceptance is not None else os.getenv("SEG_VALIDATE_ACCEPTANCE", "0") == "1"
deep_enabled = run_deep if run_deep is not None else os.getenv("SEG_VALIDATE_DEEP", "1") == "1"
real_enabled = run_real if run_real is not None else os.getenv("SEG_VALIDATE_REAL", "0") == "1"
live_enabled = run_live if run_live is not None else os.getenv("SEG_VALIDATE_LIVE", "0") == "1"
live_enabled = live_enabled or acceptance_enabled or real_enabled
if live_enabled:
backend_url = os.getenv("SEG_VALIDATE_BACKEND_URL", "http://127.0.0.1:8010")
frontend_url = os.getenv("SEG_VALIDATE_FRONTEND_URL", "http://127.0.0.1:5173")
health = _fetch(f"{backend_url}/api/health")
@@ -146,9 +160,6 @@ def validate_project(
checks.append({"name": "live_coverage_api", "passed": live_coverage["passed"] and '"task_build_passed":true' in live_coverage.get("body", "").replace(" ", ""), "detail": live_coverage})
checks.append({"name": "live_training_curves_api", "passed": live_curves["passed"] and live_curves.get("body", "").lstrip().startswith("["), "detail": live_curves})
checks.append({"name": "live_frontend_index", "passed": frontend["passed"] and "Seg Data Server" in frontend.get("body", ""), "detail": frontend})
acceptance_enabled = run_acceptance if run_acceptance is not None else os.getenv("SEG_VALIDATE_ACCEPTANCE", "1") == "1"
deep_enabled = run_deep if run_deep is not None else os.getenv("SEG_VALIDATE_DEEP", "1") == "1"
real_enabled = run_real if run_real is not None else os.getenv("SEG_VALIDATE_REAL", "0") == "1"
if acceptance_enabled:
acceptance = run_live_acceptance(backend_url)
checks.append({"name": "live_acceptance_smoke", "passed": acceptance["passed"], "detail": acceptance})
@@ -158,6 +169,9 @@ def validate_project(
if deep_enabled:
deep_acceptance = run_deep_acceptance()
checks.append({"name": "deep_training_acceptance", "passed": deep_acceptance["passed"], "detail": deep_acceptance})
elif deep_enabled:
deep_acceptance = run_deep_acceptance()
checks.append({"name": "deep_training_acceptance", "passed": deep_acceptance["passed"], "detail": deep_acceptance})
if run_build:
tests = _run(["conda", "run", "-n", settings.backend_conda_env, "python", "-m", "pytest", "-q"], cwd=settings.project_root, timeout=120)

10
backend/app/capabilities.py
View File

@@ -3,7 +3,7 @@ from __future__ import annotations
import time
from typing import Any
from .acceptance import latest_acceptance_report, latest_deep_acceptance_report
from .acceptance import latest_acceptance_report, latest_deep_acceptance_report, latest_real_acceptance_report
from .catalog import get_catalog
from .coverage import get_coverage_report
from .modules.dataset.service import list_uploaded_datasets
@@ -156,6 +156,7 @@ def get_capability_matrix() -> dict[str, Any]:
manifest = load_manifest()
gpus = get_gpus()
acceptance = latest_acceptance_report()
real_acceptance = latest_real_acceptance_report()
deep_acceptance = latest_deep_acceptance_report()
all_tasks = catalog["task_types"]
@@ -270,6 +271,12 @@ def get_capability_matrix() -> dict[str, Any]:
"passed": bool(deep_acceptance.get("passed")),
"detail": deep_acceptance.get("run_id", "not run"),
},
{
"id": "real_workspace_acceptance",
"label": "真实数据验收",
"passed": bool(real_acceptance.get("passed")),
"detail": real_acceptance.get("run_id", "not run"),
},
{
"id": "weights_manifest",
"label": "权重清单",
@@ -294,6 +301,7 @@ def get_capability_matrix() -> dict[str, Any]:
"weights": manifest.get("count", 0),
"gpus_available": bool(gpus.get("available")),
"acceptance_passed": bool(acceptance.get("passed")),
"real_acceptance_passed": bool(real_acceptance.get("passed")),
"deep_acceptance_passed": bool(deep_acceptance.get("passed")),
},
"requirements": requirements,

16
backend/app/main.py
View File

@@ -290,5 +290,17 @@ def api_agent_evaluate() -> dict:
@app.get("/api/agents/validate")
def api_agent_validate(run_build: bool = False, run_acceptance: bool = False, run_deep: bool = False, run_real: bool = False) -> dict:
return validate_project(run_build=run_build, run_acceptance=run_acceptance, run_deep=run_deep, run_real=run_real)
def api_agent_validate(
run_build: bool = False,
run_acceptance: bool = False,
run_deep: bool = False,
run_real: bool = False,
run_live: bool | None = None,
) -> dict:
return validate_project(
run_build=run_build,
run_acceptance=run_acceptance,
run_deep=run_deep,
run_real=run_real,
run_live=run_live,
)

5
backend/tests/test_agents.py
View File

@@ -5,13 +5,16 @@ from app.agents.validation_agent import validate_project
def test_evaluation_agent_returns_checks():
result = evaluate_project()
assert result["agent"] == "evaluation_suggestion_agent"
assert result["passed"] is True
assert result["checks"]
assert result["summary"]["passed_checks"] == result["summary"]["total_checks"]
checks = {item["name"]: item["passed"] for item in result["checks"]}
assert checks["real_workspace_acceptance"] is True
def test_validation_agent_lightweight(monkeypatch):
monkeypatch.setenv("SEG_VALIDATE_LIVE", "0")
result = validate_project(run_build=False)
result = validate_project(run_build=False, run_deep=False)
assert result["agent"] == "validation_agent"
assert result["passed"] is True
assert any(item["name"] == "catalog_has_yolo_heatmap" for item in result["checks"])

1
backend/tests/test_capabilities.py
View File

@@ -19,3 +19,4 @@ def test_capability_matrix_tracks_user_requirements():
assert requirements["runtime_readiness"]["passed"] is True
assert requirements["yolo_heatmap"]["passed"] is True
assert requirements["training_curves"]["passed"] is True
assert requirements["real_workspace_acceptance"]["passed"] is True

10
frontend/src/main.tsx
View File

@@ -287,7 +287,13 @@ type AgentCheck = {
type EvaluationAgentPayload = {
agent: string;
passed: boolean;
score: number;
summary?: {
passed_checks: number;
total_checks: number;
missing_checks: string[];
};
checks: AgentCheck[];
suggestions: string[];
};
@@ -1203,11 +1209,11 @@ function App() {
<p className="eyebrow">Evaluation Agent</p>
<h2>评价建议</h2>
</div>
<StatusPill status={(agentEvaluation?.score ?? 0) >= 1 ? "success" : "queued"} />
<StatusPill status={agentEvaluation?.passed ? "success" : "queued"} />
</div>
<div className="agentScore">
<strong>{Math.round((agentEvaluation?.score ?? 0) * 100)}%</strong>
<span>{agentEvaluation?.checks.filter((item) => item.passed).length ?? 0}/{agentEvaluation?.checks.length ?? 0} checks passed</span>
<span>{agentEvaluation?.summary ? `${agentEvaluation.summary.passed_checks}/${agentEvaluation.summary.total_checks}` : `${agentEvaluation?.checks.filter((item) => item.passed).length ?? 0}/${agentEvaluation?.checks.length ?? 0}`} checks passed</span>
</div>
<div className="suggestionList">
{(agentEvaluation?.suggestions ?? ["等待评价 agent 返回建议。"]).slice(0, 6).map((item, index) => (

15
scripts/run_agents.py
View File

@@ -16,20 +16,29 @@ from app.agents.validation_agent import validate_project # noqa: E402
def main() -> None:
parser = argparse.ArgumentParser(description="Run local evaluation and validation agents.")
parser.add_argument("--build", action="store_true", help="also run pytest and frontend build")
parser.add_argument("--live", action="store_true", help="also check live backend/frontend HTTP endpoints")
parser.add_argument("--acceptance", action="store_true", help="run the lightweight live acceptance smoke")
parser.add_argument("--real", action="store_true", help="run real workspace data acceptance through the live backend")
parser.add_argument("--no-deep", action="store_true", help="skip synthetic deep training acceptance")
parser.add_argument("--out", default="var/agent_reports/latest.json")
args = parser.parse_args()
report = {
"evaluation": evaluate_project(),
"validation": validate_project(run_build=args.build),
"validation": validate_project(
run_build=args.build,
run_live=args.live,
run_acceptance=args.acceptance,
run_real=args.real,
run_deep=not args.no_deep,
),
}
out = ROOT / args.out
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
if not report["validation"]["passed"]:
if not report["evaluation"]["passed"] or not report["validation"]["passed"]:
raise SystemExit(1)
if __name__ == "__main__":
main()