From 0c239483a9abb0782c66f3a141ed9223c9547fbd Mon Sep 17 00:00:00 2001 From: admin <572701190@qq.com> Date: Tue, 30 Jun 2026 14:37:02 +0800 Subject: [PATCH] Add full capability readiness matrix --- README.md | 20 +- backend/app/agents/evaluation_agent.py | 2 + backend/app/agents/validation_agent.py | 13 ++ backend/app/capabilities.py | 299 +++++++++++++++++++++++++ backend/app/main.py | 6 + backend/tests/test_capabilities.py | 21 ++ frontend/src/main.tsx | 90 +++++++- frontend/src/styles.css | 130 ++++++++++- 8 files changed, 570 insertions(+), 11 deletions(-) create mode 100644 backend/app/capabilities.py create mode 100644 backend/tests/test_capabilities.py diff --git a/README.md b/README.md index fdbf537..4eb0fd9 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,10 @@ The runtime panel calls `GET /api/system/readiness` and verifies the conda imports required for the backend/task environment and the full MMSeg/mmcv environment. Command-line verification is available with `PYTHONPATH=backend conda run -n seg_smp python scripts/verify_runtime_envs.py --refresh`. +The capability matrix calls `GET /api/capabilities` and summarizes readiness +for Dataset, SegModel, YOLO, MMSeg, visual tools, analysis, and system +operations, including task coverage, runtime status, uploaded datasets, +heatmap/segmentation artifacts, training curves, and weight manifest status. The same panel can run `POST /api/acceptance/smoke`, a lightweight live smoke that creates an upload dataset, uploads a label, downloads it through the @@ -144,6 +148,8 @@ Use `GET /api/catalog` to inspect supported models, algorithms, datasets, and task types discovered from the existing `Seg/` workspace. Use `GET /api/coverage` to inspect script-to-task coverage and task buildability. +Use `GET /api/capabilities` to inspect the grouped full-function readiness +matrix used by the web dashboard and agents. Use `GET /api/results/curves` to inspect parsed training curves discovered from YOLO, SegModel, MMSeg, visual-tool, and analysis output directories. @@ -155,10 +161,10 @@ Run the local evaluation and validation agents before publishing changes: PYTHONPATH=backend conda run -n seg_smp python scripts/run_agents.py --build ``` -The validation agent checks catalog coverage, the `seg_smp` task env, the -`seg_mmcv` MMSeg env, runtime import readiness, GPU visibility, no-weight Git -safety, backend tests, frontend build, and live backend/frontend endpoints -when the services are running. With live validation enabled it also runs the -lightweight acceptance smoke above. By default it also runs the deep training -acceptance; set `SEG_VALIDATE_DEEP=0` when a quick non-training validation -pass is needed. +The validation agent checks catalog coverage, the grouped capability matrix, +the `seg_smp` task env, the `seg_mmcv` MMSeg env, runtime import readiness, +GPU visibility, no-weight Git safety, backend tests, frontend build, and live +backend/frontend endpoints when the services are running. With live validation +enabled it also runs the lightweight acceptance smoke above. By default it +also runs the deep training acceptance; set `SEG_VALIDATE_DEEP=0` when a quick +non-training validation pass is needed. diff --git a/backend/app/agents/evaluation_agent.py b/backend/app/agents/evaluation_agent.py index 5d9096b..24186b4 100644 --- a/backend/app/agents/evaluation_agent.py +++ b/backend/app/agents/evaluation_agent.py @@ -44,10 +44,12 @@ def evaluate_project() -> dict: "loss_result_ui": "loss" in frontend_text.lower() and "heatmap" in frontend_text.lower() and "CurvePanel" in frontend_text, "job_progress_ui": "JobProgressBar" in frontend_text and "progressTrack" in frontend_text, "runtime_readiness_ui": "runtimeReadiness" in frontend_text and "环境就绪" in frontend_text, + "capability_matrix_ui": "capabilities" in frontend_text and "全功能矩阵" in frontend_text, "dataset_api": "/api/datasets" in backend_text and "api_upload_dataset_files" in backend_text, "dataset_quality_api": "/api/datasets/{dataset_name}/validate" in backend_text and "/api/datasets/{dataset_name}/yolo-yaml" in backend_text, "job_progress_api": "progress_from_log_path" in backend_text and '"progress"' in backend_text, "runtime_readiness_api": "/api/system/readiness" in backend_text, + "capability_matrix_api": "/api/capabilities" in backend_text, "runtime_bootstrap_scripts": (settings.project_root / "scripts" / "bootstrap_conda_envs.sh").exists() and (settings.project_root / "scripts" / "verify_runtime_envs.py").exists(), "curve_api": "/api/results/curves" in backend_text, diff --git a/backend/app/agents/validation_agent.py b/backend/app/agents/validation_agent.py index 05a4021..e57ab74 100644 --- a/backend/app/agents/validation_agent.py +++ b/backend/app/agents/validation_agent.py @@ -9,6 +9,7 @@ import urllib.request from pathlib import Path from ..acceptance import run_deep_acceptance, run_live_acceptance +from ..capabilities import get_capability_matrix from ..catalog import get_catalog from ..config import settings from ..coverage import get_coverage_report @@ -66,6 +67,12 @@ def validate_project(run_build: bool = False) -> dict: checks.append({"name": "mmseg_env_exists", "passed": settings.mmseg_conda_env in env_names, "detail": {"env": settings.mmseg_conda_env}}) runtime_readiness = get_runtime_readiness(force=True) checks.append({"name": "runtime_env_readiness", "passed": runtime_readiness["passed"], "detail": runtime_readiness}) + capability_matrix = get_capability_matrix() + checks.append({ + "name": "capability_matrix_ready", + "passed": capability_matrix["passed"] and capability_matrix["summary"]["ready_domains"] == capability_matrix["summary"]["total_domains"], + "detail": capability_matrix, + }) smoke = _run( [ @@ -105,6 +112,7 @@ def validate_project(run_build: bool = False) -> dict: datasets = _fetch(f"{backend_url}/api/datasets") live_jobs = _fetch(f"{backend_url}/api/jobs") live_readiness = _fetch(f"{backend_url}/api/system/readiness") + live_capabilities = _fetch(f"{backend_url}/api/capabilities") live_coverage = _fetch(f"{backend_url}/api/coverage") live_curves = _fetch(f"{backend_url}/api/results/curves") frontend = _fetch(frontend_url) @@ -124,6 +132,11 @@ def validate_project(run_build: bool = False) -> dict: "passed": live_readiness["passed"] and '"passed":true' in live_readiness.get("body", "").replace(" ", ""), "detail": live_readiness, }) + checks.append({ + "name": "live_capability_matrix_api", + "passed": live_capabilities["passed"] and '"passed":true' in live_capabilities.get("body", "").replace(" ", ""), + "detail": live_capabilities, + }) checks.append({"name": "live_coverage_api", "passed": live_coverage["passed"] and '"task_build_passed":true' in live_coverage.get("body", "").replace(" ", ""), "detail": live_coverage}) checks.append({"name": "live_training_curves_api", "passed": live_curves["passed"] and live_curves.get("body", "").lstrip().startswith("["), "detail": live_curves}) checks.append({"name": "live_frontend_index", "passed": frontend["passed"] and "Seg Data Server" in frontend.get("body", ""), "detail": frontend}) diff --git a/backend/app/capabilities.py b/backend/app/capabilities.py new file mode 100644 index 0000000..7a0cec5 --- /dev/null +++ b/backend/app/capabilities.py @@ -0,0 +1,299 @@ +from __future__ import annotations + +import time +from typing import Any + +from .acceptance import latest_acceptance_report, latest_deep_acceptance_report +from .catalog import get_catalog +from .coverage import get_coverage_report +from .modules.dataset.service import list_uploaded_datasets +from .modules.results.service import scan_results, scan_training_curves +from .modules.system.service import get_gpus, get_runtime_readiness +from .modules.weights.service import load_manifest + + +CAPABILITY_GROUPS = [ + { + "id": "dataset", + "label": "Dataset", + "description": "上传图片、label、mask，并执行预处理、配对、叠加、拼接、视频抽帧和 YOLO 数据构建。", + "required_tasks": [ + "dataset.rename", + "dataset.to_png", + "dataset.resize", + "dataset.pair", + "dataset.rebuild_labels", + "dataset.stack", + "dataset.stitch", + "dataset.video_frames", + "dataset.yolo_txt_sort", + "dataset.yolo_convert_png", + "dataset.yolo_resize", + ], + "task_prefixes": ["dataset."], + "evidence_roles": ["segmentation"], + }, + { + "id": "segmodel", + "label": "SegModel", + "description": "SMP/SegModel 单模型和批量训练、预测、raw mask、指标、FLOPs/FPS。", + "required_tasks": [ + "segmodel.train", + "segmodel.batch_train", + "segmodel.predict", + "segmodel.batch_predict", + "segmodel.flops", + "segmodel.benchmark", + "segmodel.raw_mask_check", + "segmodel.metrics", + ], + "task_prefixes": ["segmodel."], + "runtime_roles": ["backend_task"], + "families": ["segmodel"], + }, + { + "id": "yolo", + "label": "YOLO", + "description": "YOLOv8/v9/v11/v12 分割训练、上传数据集训练、预测、热度图、对比、视频预测和 raw mask。", + "required_tasks": [ + "yolo.train", + "yolo.train_custom", + "yolo.predict", + "yolo.batch_predict", + "yolo.heatmap", + "yolo.compare", + "yolo.raw_mask_check", + "yolo.video_visible", + "yolo.video_unvisible", + ], + "task_prefixes": ["yolo."], + "runtime_roles": ["backend_task"], + "families": ["yolo"], + "evidence_roles": ["heatmap", "segmentation", "curve", "video"], + }, + { + "id": "mmseg", + "label": "MMSeg", + "description": "MMSeg 数据/算法配置生成、31 个算法、训练、预测、指标、FLOPs/FPS、绘图和 loss/mIoU 提取。", + "required_tasks": [ + "mmseg.generate_data", + "mmseg.generate_alg", + "mmseg.train", + "mmseg.predict_v1", + "mmseg.predict_v2", + "mmseg.metrics", + "mmseg.flops_fps", + "mmseg.draw", + "mmseg.extract_loss_miou", + ], + "task_prefixes": ["mmseg."], + "runtime_roles": ["mmseg_full"], + "families": ["mmseg"], + }, + { + "id": "visual", + "label": "Visual Tools", + "description": "可视化工具训练、推理、FPS、YOLO11 热度图、label 转换和 8-bit PNG 生成。", + "required_tasks": [ + "visual.train", + "visual.inference", + "visual.fps", + "visual.yolo11_heatmap_v1", + "visual.yolo11_heatmap_v2", + "visual.deal_labels", + ], + "task_prefixes": ["visual."], + "runtime_roles": ["backend_task"], + "families": ["tool"], + "evidence_roles": ["heatmap", "segmentation"], + }, + { + "id": "analysis", + "label": "Analysis", + "description": "合并 SegModel/MMSeg/YOLO 指标，生成 CSV、表格、图表和性能摘要。", + "required_tasks": ["analysis.all"], + "task_prefixes": ["analysis."], + "families": ["analysis"], + }, + { + "id": "system", + "label": "System", + "description": "GPU 查询、环境检查、权重同步、任务日志/进度、取消和备份入口。", + "required_tasks": ["system.backup", "system.check_graph_card"], + "task_prefixes": ["system.", "mock."], + "runtime_roles": ["backend_task", "mmseg_full"], + }, +] + + +def _tasks_for_group(all_tasks: list[str], prefixes: list[str]) -> list[str]: + return sorted(task for task in all_tasks if any(task.startswith(prefix) for prefix in prefixes)) + + +def _artifact_matches(item: dict[str, Any], group: dict[str, Any]) -> bool: + families = set(group.get("families", [])) + roles = set(group.get("evidence_roles", [])) + return (not families or item.get("family") in families) and (not roles or item.get("role") in roles) + + +def _coverage_build_task_set(coverage: dict[str, Any]) -> set[str]: + return {item["task"] for item in coverage.get("task_build_checks", []) if item.get("passed")} + + +def _runtime_map(readiness: dict[str, Any]) -> dict[str, dict[str, Any]]: + return {item["role"]: item for item in readiness.get("envs", [])} + + +def get_capability_matrix() -> dict[str, Any]: + catalog = get_catalog() + coverage = get_coverage_report() + readiness = get_runtime_readiness(force=False) + results = scan_results(limit=1000) + curves = scan_training_curves(limit=50) + datasets = list_uploaded_datasets() + manifest = load_manifest() + gpus = get_gpus() + acceptance = latest_acceptance_report() + deep_acceptance = latest_deep_acceptance_report() + + all_tasks = catalog["task_types"] + buildable_tasks = _coverage_build_task_set(coverage) + runtime_by_role = _runtime_map(readiness) + domains = [] + + for group in CAPABILITY_GROUPS: + group_tasks = _tasks_for_group(all_tasks, group["task_prefixes"]) + required_tasks = group["required_tasks"] + missing_required = [task for task in required_tasks if task not in all_tasks] + unbuildable_required = [task for task in required_tasks if task in all_tasks and task not in buildable_tasks] + runtime_roles = group.get("runtime_roles", []) + runtime_reports = [runtime_by_role.get(role) for role in runtime_roles if runtime_by_role.get(role)] + runtime_ready = all(item.get("passed") for item in runtime_reports) if runtime_roles else True + artifacts = [item for item in results if _artifact_matches(item, group)] + group_curves = [item for item in curves if not group.get("families") or item.get("family") in group["families"]] + evidence_count = len(artifacts) + len(group_curves) + + if group["id"] == "dataset": + evidence_count += sum(dataset["counts"]["images"] + dataset["counts"]["labels"] + dataset["counts"]["masks"] for dataset in datasets) + if group["id"] == "mmseg": + evidence_count += len(catalog.get("mmseg_algorithms", [])) + if group["id"] == "system": + evidence_count += int(bool(gpus.get("available"))) + int(manifest.get("count", 0) > 0) + + gaps = [] + if missing_required: + gaps.append(f"missing tasks: {', '.join(missing_required[:4])}") + if unbuildable_required: + gaps.append(f"unbuildable tasks: {', '.join(unbuildable_required[:4])}") + if runtime_roles and not runtime_ready: + gaps.append("runtime env check failed") + + domains.append( + { + "id": group["id"], + "label": group["label"], + "description": group["description"], + "ready": not missing_required and not unbuildable_required and runtime_ready, + "tasks": { + "total": len(group_tasks), + "required": len(required_tasks), + "required_ready": len(required_tasks) - len(missing_required) - len(unbuildable_required), + "examples": group_tasks[:10], + "missing_required": missing_required, + "unbuildable_required": unbuildable_required, + }, + "runtime": [ + {"role": item["role"], "name": item["name"], "passed": item["passed"]} + for item in runtime_reports + ], + "evidence": { + "count": evidence_count, + "artifacts": [ + { + "name": item["name"], + "relative_path": item["relative_path"], + "role": item.get("role"), + "family": item.get("family"), + } + for item in artifacts[:6] + ], + "curves": [ + { + "name": item["name"], + "relative_path": item["relative_path"], + "family": item.get("family"), + "row_count": item.get("row_count"), + } + for item in group_curves[:4] + ], + }, + "gaps": gaps, + } + ) + + requirements = [ + { + "id": "user_script_mapping", + "label": "用户侧脚本映射", + "passed": coverage["mapped_user_scripts"] == coverage["user_scripts_total"] and not coverage["unmapped_user_scripts"], + "detail": f"{coverage['mapped_user_scripts']}/{coverage['user_scripts_total']}", + }, + { + "id": "runtime_readiness", + "label": "运行环境就绪", + "passed": readiness.get("passed", False), + "detail": ", ".join(f"{item['name']}={item['passed']}" for item in readiness.get("envs", [])), + }, + { + "id": "dataset_upload", + "label": "数据集上传/Label/Mask", + "passed": len(datasets) >= 1, + "detail": f"{len(datasets)} uploaded dataset(s)", + }, + { + "id": "yolo_heatmap", + "label": "YOLO 热度图证据", + "passed": any(item.get("family") == "yolo" and item.get("role") == "heatmap" for item in results), + "detail": "heatmap artifact discovered", + }, + { + "id": "training_curves", + "label": "训练 loss/指标曲线", + "passed": len(curves) >= 1, + "detail": f"{len(curves)} curve file(s)", + }, + { + "id": "deep_acceptance", + "label": "深度训练验收", + "passed": bool(deep_acceptance.get("passed")), + "detail": deep_acceptance.get("run_id", "not run"), + }, + { + "id": "weights_manifest", + "label": "权重清单", + "passed": manifest.get("count", 0) >= 1, + "detail": f"{manifest.get('count', 0)} weights indexed", + }, + ] + + ready_domains = sum(1 for item in domains if item["ready"]) + return { + "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "passed": ready_domains == len(domains) and all(item["passed"] for item in requirements), + "summary": { + "ready_domains": ready_domains, + "total_domains": len(domains), + "mapped_user_scripts": coverage["mapped_user_scripts"], + "user_scripts_total": coverage["user_scripts_total"], + "task_build_passed": coverage["task_build_passed"], + "uploaded_datasets": len(datasets), + "artifacts": len(results), + "curves": len(curves), + "weights": manifest.get("count", 0), + "gpus_available": bool(gpus.get("available")), + "acceptance_passed": bool(acceptance.get("passed")), + "deep_acceptance_passed": bool(deep_acceptance.get("passed")), + }, + "requirements": requirements, + "domains": domains, + } diff --git a/backend/app/main.py b/backend/app/main.py index 6823c3b..739de74 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -10,6 +10,7 @@ from fastapi.responses import FileResponse, StreamingResponse from . import db from .acceptance import latest_acceptance_report, latest_deep_acceptance_report, run_deep_acceptance, run_live_acceptance +from .capabilities import get_capability_matrix from .catalog import get_catalog from .config import settings from .coverage import get_coverage_report @@ -86,6 +87,11 @@ def api_coverage() -> dict: return get_coverage_report() +@app.get("/api/capabilities") +def api_capabilities() -> dict: + return get_capability_matrix() + + @app.get("/api/acceptance/latest") def api_acceptance_latest() -> dict: return latest_acceptance_report() diff --git a/backend/tests/test_capabilities.py b/backend/tests/test_capabilities.py new file mode 100644 index 0000000..425d86c --- /dev/null +++ b/backend/tests/test_capabilities.py @@ -0,0 +1,21 @@ +from app.capabilities import get_capability_matrix + + +def test_capability_matrix_covers_core_domains(): + matrix = get_capability_matrix() + domains = {item["id"]: item for item in matrix["domains"]} + + assert {"dataset", "segmodel", "yolo", "mmseg", "visual", "analysis", "system"} <= set(domains) + assert domains["dataset"]["tasks"]["required_ready"] == domains["dataset"]["tasks"]["required"] + assert domains["yolo"]["ready"] is True + assert domains["mmseg"]["ready"] is True + + +def test_capability_matrix_tracks_user_requirements(): + matrix = get_capability_matrix() + requirements = {item["id"]: item for item in matrix["requirements"]} + + assert requirements["user_script_mapping"]["passed"] is True + assert requirements["runtime_readiness"]["passed"] is True + assert requirements["yolo_heatmap"]["passed"] is True + assert requirements["training_curves"]["passed"] is True diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx index a6be61f..ca566dc 100644 --- a/frontend/src/main.tsx +++ b/frontend/src/main.tsx @@ -193,6 +193,44 @@ type RuntimeReadinessPayload = { }; }; +type CapabilityDomain = { + id: string; + label: string; + ready: boolean; + tasks: { + total: number; + required: number; + required_ready: number; + examples: string[]; + missing_required: string[]; + unbuildable_required: string[]; + }; + runtime: Array<{ role: string; name: string; passed: boolean }>; + evidence: { + count: number; + artifacts: Array<{ name: string; relative_path: string; role?: string; family?: string }>; + curves: Array<{ name: string; relative_path: string; family?: string; row_count?: number }>; + }; + gaps: string[]; +}; + +type CapabilityPayload = { + passed: boolean; + generated_at: string; + summary: { + ready_domains: number; + total_domains: number; + mapped_user_scripts: number; + user_scripts_total: number; + uploaded_datasets: number; + artifacts: number; + curves: number; + weights: number; + }; + requirements: Array<{ id: string; label: string; passed: boolean; detail: string }>; + domains: CapabilityDomain[]; +}; + async function api(path: string, init?: RequestInit): Promise { const res = await fetch(`${API_BASE}${path}`, { headers: { "Content-Type": "application/json" }, @@ -266,14 +304,16 @@ function useData() { const [acceptance, setAcceptance] = useState(null); const [deepAcceptance, setDeepAcceptance] = useState(null); const [runtimeReadiness, setRuntimeReadiness] = useState(null); + const [capabilities, setCapabilities] = useState(null); const [error, setError] = useState(""); async function refresh() { try { - const [catalogNext, gpusNext, readinessNext, jobsNext, resultsNext, curvesNext, datasetsNext, coverageNext, acceptanceNext, deepAcceptanceNext] = await Promise.all([ + const [catalogNext, gpusNext, readinessNext, capabilitiesNext, jobsNext, resultsNext, curvesNext, datasetsNext, coverageNext, acceptanceNext, deepAcceptanceNext] = await Promise.all([ api("/api/catalog"), api("/api/system/gpus"), api("/api/system/readiness"), + api("/api/capabilities"), api("/api/jobs"), api("/api/results"), api("/api/results/curves"), @@ -285,6 +325,7 @@ function useData() { setCatalog(catalogNext); setGpus(gpusNext); setRuntimeReadiness(readinessNext); + setCapabilities(capabilitiesNext); setJobs(jobsNext); setResults(resultsNext.slice(0, 80)); setCurves(curvesNext.slice(0, 12)); @@ -316,7 +357,7 @@ function useData() { return () => window.clearInterval(timer); }, []); - return { catalog, gpus, runtimeReadiness, jobs, results, curves, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh }; + return { catalog, gpus, runtimeReadiness, capabilities, jobs, results, curves, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh }; } function StatusPill({ status }: { status: string }) { @@ -339,7 +380,7 @@ function JobProgressBar({ progress }: { progress?: JobProgress }) { } function App() { - const { catalog, gpus, runtimeReadiness, jobs, results, curves, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh } = useData(); + const { catalog, gpus, runtimeReadiness, capabilities, jobs, results, curves, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh } = useData(); const [taskType, setTaskType] = useState("mock.echo"); const [params, setParams] = useState(JSON.stringify(defaultParams["mock.echo"], null, 2)); const [selectedJob, setSelectedJob] = useState(null); @@ -529,6 +570,7 @@ function App() {